<commit_before>package main

import (
	"bytes"
	"encoding/gob"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"text/template"

	"github.com/Masterminds/sprig"
	"gopkg.in/yaml.v2"
	"k8s.io/apimachinery/pkg/util/sets"

	rt "github.com/kyma-project/test-infra/development/tools/pkg/rendertemplates"
)

const (
	// autogenerationMessage is message added at the beginning of each autogenerated file.
	autogenerationMessage = "Code generated by rendertemplates. DO NOT EDIT."
)

var (
	configFilePath = flag.String("config", "", "Path of the config file")
	showOutputDir  = flag.Bool("show-output-dir", false, "Print generated output file paths to stdout")

	additionalFuncs = map[string]interface{}{
		"matchingReleases": rt.MatchingReleases,
		"releaseMatches":   rt.ReleaseMatches,
		"hasPresubmit":     hasPresubmit,
		"hasPostsubmit":    hasPostsubmit,
		"hasPeriodic":      hasPeriodic,
		"getRunId":         getRunID,
	}
	commentSignByFileExt = map[string]sets.String{
		"//": sets.NewString(".go"),
		"> ": sets.NewString(".md"),
		"#":  sets.NewString(".yaml", ".yml"),
	}
)

func init() {
	gob.Register(map[string]interface{}{})
	gob.Register(map[interface{}]interface{}{})
	gob.Register([]interface{}{})
}

func main() {
	flag.Parse()

	if *configFilePath == "" {
		log.Fatal("Provide path to config file with --config")
	}

	// read template config file containing global configsets
	configFile, err := ioutil.ReadFile(*configFilePath)
	if err != nil {
		log.Fatalf("Cannot read config file: %s", err)
	}

	config := new(rt.Config)
	err = yaml.Unmarshal(configFile, config)
	if err != nil {
		log.Fatalf("Cannot parse config yaml: %s\n", err)
	}

	dataFilesDir := filepath.Join(filepath.Dir(*configFilePath), "data")
	// read all template data from data files
	dataFiles, err := ioutil.ReadDir(dataFilesDir)
	if err != nil {
		log.Fatalf("Cannot read data file directory: %s", err)
	}

	var dataFilesTemplates []*rt.TemplateConfig
	for _, dataFile := range dataFiles {
		if !dataFile.IsDir() {
			var dataFileConfig rt.Config
			var cfg bytes.Buffer
			// load datafile as template
			t, err := loadTemplate(dataFilesDir, dataFile.Name())
			if err != nil {
				log.Fatalf("Could not load data file %s: %v", dataFile.Name(), err)
			}
			// execute rendering the datafile from template and store it in-memory
			// at this point the config has all the global values from config.yaml file
			if err := t.Execute(&cfg, config); err != nil {
				log.Fatalf("Cannot render data template: %v", err)
			}
			if err := yaml.Unmarshal(cfg.Bytes(), &dataFileConfig); err != nil {
				log.Fatalf("Cannot parse data file yaml: %s\n", err)
			}
			dataFilesTemplates = append(dataFilesTemplates, dataFileConfig.Templates...)
		}

	}

	//append all generated configs from datafiles to the list of templates to generate jobs from
	config.Templates = append(config.Templates, dataFilesTemplates...)
	config.Merge()

	// generate final .yaml files
	for _, templateConfig := range config.Templates {
		err = renderTemplate(path.Dir(*configFilePath), templateConfig, config)
		if err != nil {
			log.Fatalf("Cannot render template %s: %s", templateConfig.From, err)
		}
	}
}

// renderTemplate loads the template and calls the function that renders final files
func renderTemplate(basePath string, templateConfig *rt.TemplateConfig, config *rt.Config) error {
	for _, fromTo := range templateConfig.FromTo {
		if *showOutputDir {
			log.Printf("Rendering %s", fromTo)
		}
		templateInstance, err := loadTemplate(basePath, fromTo.From)
		if err != nil {
			return err
		}
		for _, render := range templateConfig.Render {
			err = renderFileFromTemplate(basePath, templateInstance, *render, config, fromTo)<commit_msg>make rendertemplates work with recursive data files (#4817)<commit_after>package main

import (
	"bytes"
	"encoding/gob"
	"flag"
	"fmt"
	"io/fs"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/Masterminds/sprig"
	"gopkg.in/yaml.v2"
	"k8s.io/apimachinery/pkg/util/sets"

	rt "github.com/kyma-project/test-infra/development/tools/pkg/rendertemplates"
)

const (
	// autogenerationMessage is message added at the beginning of each autogenerated file.
	autogenerationMessage = "Code generated by rendertemplates. DO NOT EDIT."
)

var (
	configFilePath = flag.String("config", "", "Path of the config file")
	showOutputDir  = flag.Bool("show-output-dir", false, "Print generated output file paths to stdout")

	additionalFuncs = map[string]interface{}{
		"matchingReleases": rt.MatchingReleases,
		"releaseMatches":   rt.ReleaseMatches,
		"hasPresubmit":     hasPresubmit,
		"hasPostsubmit":    hasPostsubmit,
		"hasPeriodic":      hasPeriodic,
		"getRunId":         getRunID,
	}
	commentSignByFileExt = map[string]sets.String{
		"//": sets.NewString(".go"),
		"> ": sets.NewString(".md"),
		"#":  sets.NewString(".yaml", ".yml"),
	}
)

func init() {
	gob.Register(map[string]interface{}{})
	gob.Register(map[interface{}]interface{}{})
	gob.Register([]interface{}{})
}

func main() {
	flag.Parse()

	if *configFilePath == "" {
		log.Fatal("Provide path to config file with --config")
	}

	// read template config file containing global configsets
	configFile, err := ioutil.ReadFile(*configFilePath)
	if err != nil {
		log.Fatalf("Cannot read config file: %s", err)
	}

	config := new(rt.Config)
	err = yaml.Unmarshal(configFile, config)
	if err != nil {
		log.Fatalf("Cannot parse config yaml: %s\n", err)
	}

	dataFilesDir := filepath.Join(filepath.Dir(*configFilePath), "data")
	// read all template data from data files
	var dataFiles []string
	err = filepath.Walk(dataFilesDir, getFileWalkFunc(dataFilesDir, &dataFiles))
	if err != nil {
		log.Fatalf("Cannot read data file directory: %s", err)
	}

	var dataFilesTemplates []*rt.TemplateConfig
	for _, dataFile := range dataFiles {
		var dataFileConfig rt.Config
		var cfg bytes.Buffer
		// load datafile as template
		t, err := loadTemplate(dataFilesDir, dataFile)
		if err != nil {
			log.Fatalf("Could not load data file %s: %v", dataFile, err)
		}
		// execute rendering the datafile from template and store it in-memory
		// at this point the config has all the global values from config.yaml file
		if err := t.Execute(&cfg, config); err != nil {
			log.Fatalf("Cannot render data template: %v", err)
		}
		if err := yaml.Unmarshal(cfg.Bytes(), &dataFileConfig); err != nil {
			log.Fatalf("Cannot parse data file yaml: %s\n", err)
		}
		dataFilesTemplates = append(dataFilesTemplates, dataFileConfig.Templates...)
	}

	//append all generated configs from datafiles to the list of templates to generate jobs from
	config.Templates = append(config.Templates, dataFilesTemplates...)
	config.Merge()

	// generate final .yaml files
	for _, templateConfig := range config.Templates {
		err = renderTemplate(path.Dir(*configFilePath), templateConfig, config)
		if err != nil {
			log.Fatalf("Cannot render template %s: %s", templateConfig.From, err)
		}
	}
}

// getFileWalkFunc returns walk function that will recursively find YAML files and will return list of these files
func getFileWalkFunc(dataFilesDir string, dataFiles *[]string) filepath.WalkFunc {
	return func(path string, info fs.FileInfo, err error) error {
		//pass the error further, this shouldn't ever happen
		if err != nil {
			return err
		}

		// skip directory entries, we just want files
		if info.IsDir() {
			return nil
		}

		// we only want to check .yaml files
		if !strings.Contains(info.Name(), ".yaml") {
			return nil
		}

		// get relative path
		dataFile := strings.Replace(path, dataFilesDir, "", -1)
		// add all YAML files to the list
		*dataFiles = append(*dataFiles, dataFile)

		return nil
	}
}

// renderTemplate loads the template and calls the function that renders final files
func renderTemplate(basePath string, templateConfig *rt.TemplateConfig, config *rt.Config) error {
	for _, fromTo := range templateConfig.FromTo {
		if *showOutputDir {
			log.Printf("Rendering %s", fromTo)
		}
		templateInstance, err := loadTemplate(basePath, fromTo.From)
		if err != nil {
			return err
		}
		for _, render := range templateConfig.Render {
			err = renderFileFromTemplate(basePath, templateInstance, *render, config, fromTo)
{'fraction_non_alphanumeric': 0.1301918872965752, 'fraction_numerical': 0.0010930289045421424, 'mean_word_length': 4.083333333333333, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2757548', 'n_tokens_mistral': 2821, 'n_tokens_neox': 2631, 'n_words': 744}