<commit_before>package main

import (
	"encoding/xml"
	"flag"
	"log"
	"os"
	"strings"
	"ftw.at/dbstream/server/utils"
)

type JobDefinition struct {
	Description string `xml:"description,attr"`
	Inputs      string `xml:"inputs,attr"`
	Output      string `xml:"output,attr"`
	Priority    int    `xml:"priority,attr"`
	StartTime   int64  `xml:"startTime,attr"`
	Schema      string `xml:"schema,attr"`
	Index		string `xml:"index,attr"`
	Query       string `xml:"query"`
}

type extTable struct {
	XMLName xml.Name `xml:"table"`
	Name string `xml:"name,attr"`
}

type jobNode struct {
	inputs []string
	name string
}

type Config struct {
	XMLName    xml.Name `xml:"config"`
	ExtImports []extTable `xml:"modules>module>config>tables>table"`
	Jobs       []JobDefinition `xml:"modules>module>config>jobs>job"`
}

var configFileName = flag.String("config", "", "The configuration file used.")
var format = flag.String("format", "gv", "The fomat to plot the graph in, e.g. gexf or gv.")
var stlye = flag.String("style", "id", "The style of the graph labels e.g. name, id or name_id only.")

func readConfig() (cfg Config) {
	//decode the xml config file
	cfgFile, err := os.Open(*configFileName)
	if err != nil {
		log.Fatalf("ERROR while reading config: %v\n", err)
	}
	decode := xml.NewDecoder(cfgFile)
	err = decode.Decode(&cfg)
	if err != nil {
		log.Fatalf("ERROR while decoding config: %v\n", err)
	}
	cfgFile.Close()
	return
}

func csvToSlice(csv string) (out []string) {
	if strings.Index(csv, ",") > 0 {
		out = strings.Split(csv, ",")
		for i := 0; i < len(out); i++ {
			out[i] = strings.Trim(out[i], " \t")
		}
	} else {
		if len(csv) > 0 {
			out = make([]string, 1)
			out[0] = strings.Trim(csv, " \t")
		}
	}
	return out
}

func jobDef2node(jd JobDefinition) (jout jobNode) {
	inputs := csvToSlice(jd.Inputs)
	jout.inputs = make([]string, len(inputs))
	for i, inp := range inputs {
		jout.inputs[i] = utils.IOWindowFromString(inp).Name
	}

	jout.name = utils.IOWindowFromString(jd.Output).Name
	return jout
}


func main() {
	flag.Parse()
	log.SetFlags(19)
	cfg := readConfig()

	totalLen := len(cfg.Jobs) + len(cfg.ExtImports)

	jobs := make([]jobNode, totalLen)

	//ExtImports to jobNodes
	i := 0
	for _, tbl := range cfg.ExtImports {
		 jobs[i] = jobNode{name: tbl.Name}
		 i++
	}
	//jobs to jobNodes
	for _, job := range cfg.Jobs {
		 jobs[i] = jobDef2node(job)<commit_msg>IOWindow updated<commit_after>package main

import (
	"encoding/xml"
	"flag"
	"log"
	"os"
	"strings"
	"types"
)

type JobDefinition struct {
	Description string `xml:"description,attr"`
	Inputs      string `xml:"inputs,attr"`
	Output      string `xml:"output,attr"`
	Priority    int    `xml:"priority,attr"`
	StartTime   int64  `xml:"startTime,attr"`
	Schema      string `xml:"schema,attr"`
	Index		string `xml:"index,attr"`
	Query       string `xml:"query"`
}

type extTable struct {
	XMLName xml.Name `xml:"table"`
	Name string `xml:"name,attr"`
}

type jobNode struct {
	inputs []string
	name string
}

type Config struct {
	XMLName    xml.Name `xml:"config"`
	ExtImports []extTable `xml:"modules>module>config>tables>table"`
	Jobs       []JobDefinition `xml:"modules>module>config>jobs>job"`
}

var configFileName = flag.String("config", "", "The configuration file used.")
var format = flag.String("format", "gv", "The fomat to plot the graph in, e.g. gexf or gv.")
var stlye = flag.String("style", "id", "The style of the graph labels e.g. name, id or name_id only.")

func readConfig() (cfg Config) {
	//decode the xml config file
	cfgFile, err := os.Open(*configFileName)
	if err != nil {
		log.Fatalf("ERROR while reading config: %v\n", err)
	}
	decode := xml.NewDecoder(cfgFile)
	err = decode.Decode(&cfg)
	if err != nil {
		log.Fatalf("ERROR while decoding config: %v\n", err)
	}
	cfgFile.Close()
	return
}

func csvToSlice(csv string) (out []string) {
	if strings.Index(csv, ",") > 0 {
		out = strings.Split(csv, ",")
		for i := 0; i < len(out); i++ {
			out[i] = strings.Trim(out[i], " \t")
		}
	} else {
		if len(csv) > 0 {
			out = make([]string, 1)
			out[0] = strings.Trim(csv, " \t")
		}
	}
	return out
}

func jobDef2node(jd JobDefinition) (jout jobNode) {
	inputs := csvToSlice(jd.Inputs)
	jout.inputs = make([]string, len(inputs))
	for i, inp := range inputs {
		jout.inputs[i] = types.IOWindowFromString(inp).Name
	}

	jout.name = types.IOWindowFromString(jd.Output).Name
	return jout
}


func main() {
	flag.Parse()
	log.SetFlags(19)
	cfg := readConfig()

	totalLen := len(cfg.Jobs) + len(cfg.ExtImports)

	jobs := make([]jobNode, totalLen)

	//ExtImports to jobNodes
	i := 0
	for _, tbl := range cfg.ExtImports {
		 jobs[i] = jobNode{name: tbl.Name}
		 i++
	}
	//jobs to jobNodes
	for _, job := range cfg.Jobs {
		 jobs[i] = jobDef2node(job)
{'fraction_non_alphanumeric': 0.16336842105263158, 'fraction_numerical': 0.0050526315789473685, 'mean_word_length': 3.87782340862423, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 36}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6922948', 'n_tokens_mistral': 1825, 'n_tokens_neox': 1748, 'n_words': 448}