<commit_before>package main

import (
	"crypto/rand"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"log"
)

type configFile struct {
	Listen               string
	Threads              int
	Workers              int
	Retries              int
	APITimeout           int
	CacheDir             string
	FastStart            bool
	Secret               string `xml:",omitempty"`
	ProxyAddr            string `xml:",omitempty"`
	RealRemoteAddrHeader string `xml:",omitempty"`
	Logging              logConfig
}

type logConfig struct {
	LogFile string

	Debug        bool
	DebugLogFile string

	LogRequests bool
	CensorLog   bool
}

var conf configFile

func genSecret() string {
	buf := make([]byte, 32)
	io.ReadFull(rand.Reader, buf)
	return fmt.Sprintf("%x", buf)
}

var defaultConfig = configFile{
	Listen:     "127.0.0.1:3748",
	Threads:    0,
	Workers:    10,
	Retries:    3,
	APITimeout: 60,
	CacheDir:   "cache/",
	Logging: logConfig{
		CensorLog: true,
	},
}

func createConfig() {
	//  Secret for eventual control code
	//	defaultConfig.Secret = genSecret()
	confXML, _ := xml.MarshalIndent(defaultConfig, "", "  ")

	err := ioutil.WriteFile("apiproxy.xml.default", confXML, 0600)
	if err != nil {
		log.Fatalf("Error creating config file apiproxy.xml.default: %s", err)
	} else {
		log.Fatalf("Created new config file apiproxy.xml.default")
	}
}

func loadConfig(filename string) (configFile, error) {
	conf, err := ioutil.ReadFile(filename)
	if err != nil {
		return defaultConfig, err
	}

	newConfig := defaultConfig
	err = xml.Unmarshal(conf, &newConfig)
	if err != nil {
		return defaultConfig, err
	}

	if newConfig.CacheDir == "" {
		return defaultConfig, fmt.Errorf("Need cache directory")
	}

	if newConfig.Logging.Debug {
		debug = true
	}

	return newConfig, nil
}
<commit_msg>minor order change<commit_after>package main

import (
	"crypto/rand"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"log"
)

type configFile struct {
	Listen               string
	Threads              int
	Workers              int
	Retries              int
	APITimeout           int
	CacheDir             string
	FastStart            bool
	Secret               string `xml:",omitempty"`
	ProxyAddr            string `xml:",omitempty"`
	RealRemoteAddrHeader string `xml:",omitempty"`
	Logging              logConfig
}

type logConfig struct {
	LogFile string

	LogRequests bool
	CensorLog   bool

	Debug        bool
	DebugLogFile string
}

var conf configFile

func genSecret() string {
	buf := make([]byte, 32)
	io.ReadFull(rand.Reader, buf)
	return fmt.Sprintf("%x", buf)
}

var defaultConfig = configFile{
	Listen:     "127.0.0.1:3748",
	Threads:    0,
	Workers:    10,
	Retries:    3,
	APITimeout: 60,
	CacheDir:   "cache/",
	Logging: logConfig{
		CensorLog: true,
	},
}

func createConfig() {
	//  Secret for eventual control code
	//	defaultConfig.Secret = genSecret()
	confXML, _ := xml.MarshalIndent(defaultConfig, "", "  ")

	err := ioutil.WriteFile("apiproxy.xml.default", confXML, 0600)
	if err != nil {
		log.Fatalf("Error creating config file apiproxy.xml.default: %s", err)
	} else {
		log.Fatalf("Created new config file apiproxy.xml.default")
	}
}

func loadConfig(filename string) (configFile, error) {
	conf, err := ioutil.ReadFile(filename)
	if err != nil {
		return defaultConfig, err
	}

	newConfig := defaultConfig
	err = xml.Unmarshal(conf, &newConfig)
	if err != nil {
		return defaultConfig, err
	}

	if newConfig.CacheDir == "" {
		return defaultConfig, fmt.Errorf("Need cache directory")
	}

	if newConfig.Logging.Debug {
		debug = true
	}

	return newConfig, nil
}

{'fraction_non_alphanumeric': 0.11363000279876855, 'fraction_numerical': 0.012314581584102995, 'mean_word_length': 2.863783783783784, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1480697', 'n_tokens_mistral': 1260, 'n_tokens_neox': 1202, 'n_words': 308}