<commit_before>package env_strings

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

func newFuncMap() map[string]interface{} {
	m := make(map[string]interface{})
	m["base"] = path.Base
	m["abs"] = filepath.Abs
	m["dir"] = filepath.Dir
	m["pwd"] = os.Getwd
	m["split"] = strings.Split
	m["json"] = UnmarshalJsonObject
	m["jsonArray"] = UnmarshalJsonArray
	m["dir"] = path.Dir
	m["getenv"] = os.Getenv
	m["join"] = strings.Join
	m["localtime"] = time.Now
	m["utc"] = time.Now().UTC
	m["pid"] = os.Getpid
	m["httpGet"] = httpGet

	return m
}

func UnmarshalJsonObject(data string) (map[string]interface{}, error) {
	var ret map[string]interface{}
	err := json.Unmarshal([]byte(data), &ret)
	return ret, err
}

func UnmarshalJsonArray(data string) ([]interface{}, error) {
	var ret []interface{}
	err := json.Unmarshal([]byte(data), &ret)
	return ret, err
}

func httpGet(url string) (ret string, err error) {
	var resp *http.Response
	if resp, err = http.Get(url); err != nil {
		return
	}

	var body []byte
	if body, err = ioutil.ReadAll(resp.Body); err != nil {
		return
	}

	ret = string(body)

	return
}<commit_msg>add envIfElse<commit_after>package env_strings

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

func newFuncMap() map[string]interface{} {
	m := make(map[string]interface{})
	m["base"] = path.Base
	m["abs"] = filepath.Abs
	m["dir"] = filepath.Dir
	m["pwd"] = os.Getwd
	m["split"] = strings.Split
	m["json"] = UnmarshalJsonObject
	m["jsonArray"] = UnmarshalJsonArray
	m["dir"] = path.Dir
	m["getenv"] = os.Getenv
	m["join"] = strings.Join
	m["localtime"] = time.Now
	m["utc"] = time.Now().UTC
	m["pid"] = os.Getpid
	m["httpGet"] = httpGet
	m["envIfElse"] = envIfElse

	return m
}

func UnmarshalJsonObject(data string) (map[string]interface{}, error) {
	var ret map[string]interface{}
	err := json.Unmarshal([]byte(data), &ret)
	return ret, err
}

func UnmarshalJsonArray(data string) ([]interface{}, error) {
	var ret []interface{}
	err := json.Unmarshal([]byte(data), &ret)
	return ret, err
}

func httpGet(url string) (ret string, err error) {
	var resp *http.Response
	if resp, err = http.Get(url); err != nil {
		return
	}

	var body []byte
	if body, err = ioutil.ReadAll(resp.Body); err != nil {
		return
	}

	ret = string(body)

	return
}

func envIfElse(envName, equalValue, trueValue, falseValue string) string {
	if os.Getenv(envName) == equalValue {
		return trueValue
	} else {
		return falseValue
	}
}
{'fraction_non_alphanumeric': 0.17670208579299487, 'fraction_numerical': 0.0, 'mean_word_length': 4.51409978308026, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5786318', 'n_tokens_mistral': 1007, 'n_tokens_neox': 975, 'n_words': 226}