<commit_before>// Copyright (c) 2016 Pulcy.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package jobs

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"

	"github.com/juju/errgo"

	"github.com/op/go-logging"
	"github.com/pulcy/j2/cluster"
	fg "github.com/pulcy/j2/flags"
	"github.com/pulcy/j2/pkg/vault"
)

const (
	privateLoadBalancerPort    = 81
	privateTcpLoadBalancerPort = 82
)

type jobFunctions struct {
	jobPath string
	options fg.Options
	cluster cluster.Cluster
	log     *logging.Logger
	vault.VaultConfig
	vault.GithubLoginData
}

// newJobFunctions creates a new instance of jobFunctions
func newJobFunctions(jobPath string, cluster cluster.Cluster, options fg.Options,
	log *logging.Logger, vaultConfig vault.VaultConfig, ghLoginData vault.GithubLoginData) *jobFunctions {
	absJobPath, _ := filepath.Abs(jobPath)
	return &jobFunctions{
		jobPath:         absJobPath,
		options:         options,
		cluster:         cluster,
		VaultConfig:     vaultConfig,
		GithubLoginData: ghLoginData,
		log:             log,
	}
}

// Functions returns all supported template functions
func (jf *jobFunctions) Functions() template.FuncMap {
	return template.FuncMap{
		"cat":          jf.cat,
		"env":          jf.getEnv,
		"opt":          jf.getOpt,
		"quote":        strconv.Quote,
		"replace":      strings.Replace,
		"trim":         strings.TrimSpace,
		"private_ipv4": func() string { return "${COREOS_PRIVATE_IPV4}" },
		"public_ipv4":  func() string { return "${COREOS_PUBLIC_IPV4}" },
		"hostname":     func() string { return "%H" },
		"machine_id":   func() string { return "%m" },
		"link_url":     linkURL,
		"link_tcp":     linkTCP,
		"link_tls":     linkTLS,
		"secret":       jf.vaultExtract,
		"include":      jf.include,
	}
}

func (jf *jobFunctions) Options() parseJobOptions {
	return parseJobOptions{
		Cluster: jf.cluster,
	}
}

// getEnv loads an environment value and returns an error if it is empty.
func (jf *jobFunctions) getEnv(key string) (string, error) {
	value := os.Getenv(key)
	if value == "" {
		return "", errgo.WithCausef(nil, ValidationError, "Missing environment variables '%s'", key)
	}
	return value, nil
}<commit_msg>Added string `split` template function<commit_after>// Copyright (c) 2016 Pulcy.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package jobs

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"

	"github.com/juju/errgo"

	"github.com/op/go-logging"
	"github.com/pulcy/j2/cluster"
	fg "github.com/pulcy/j2/flags"
	"github.com/pulcy/j2/pkg/vault"
)

const (
	privateLoadBalancerPort    = 81
	privateTcpLoadBalancerPort = 82
)

type jobFunctions struct {
	jobPath string
	options fg.Options
	cluster cluster.Cluster
	log     *logging.Logger
	vault.VaultConfig
	vault.GithubLoginData
}

// newJobFunctions creates a new instance of jobFunctions
func newJobFunctions(jobPath string, cluster cluster.Cluster, options fg.Options,
	log *logging.Logger, vaultConfig vault.VaultConfig, ghLoginData vault.GithubLoginData) *jobFunctions {
	absJobPath, _ := filepath.Abs(jobPath)
	return &jobFunctions{
		jobPath:         absJobPath,
		options:         options,
		cluster:         cluster,
		VaultConfig:     vaultConfig,
		GithubLoginData: ghLoginData,
		log:             log,
	}
}

// Functions returns all supported template functions
func (jf *jobFunctions) Functions() template.FuncMap {
	return template.FuncMap{
		"cat":          jf.cat,
		"env":          jf.getEnv,
		"opt":          jf.getOpt,
		"quote":        strconv.Quote,
		"replace":      strings.Replace,
		"trim":         strings.TrimSpace,
		"split":        strings.Split,
		"private_ipv4": func() string { return "${COREOS_PRIVATE_IPV4}" },
		"public_ipv4":  func() string { return "${COREOS_PUBLIC_IPV4}" },
		"hostname":     func() string { return "%H" },
		"machine_id":   func() string { return "%m" },
		"link_url":     linkURL,
		"link_tcp":     linkTCP,
		"link_tls":     linkTLS,
		"secret":       jf.vaultExtract,
		"include":      jf.include,
	}
}

func (jf *jobFunctions) Options() parseJobOptions {
	return parseJobOptions{
		Cluster: jf.cluster,
	}
}

// getEnv loads an environment value and returns an error if it is empty.
func (jf *jobFunctions) getEnv(key string) (string, error) {
	value := os.Getenv(key)
	if value == "" {
		return "", errgo.WithCausef(nil, ValidationError, "Missing environment variables '%s'", key)
	}
	return value, nil
}
{'fraction_non_alphanumeric': 0.1185360524399126, 'fraction_numerical': 0.006919155134741442, 'mean_word_length': 3.433414043583535, 'pattern_counts': {'":': 31, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1256268', 'n_tokens_mistral': 1868, 'n_tokens_neox': 1791, 'n_words': 514}