<commit_before>package main

import (
    "bufio"
    "fmt"
    "io"
    "log"
    "os"
    "regexp"
    "gopkg.in/alecthomas/kingpin.v1"
)

var (
    app = kingpin.New("substenv", "Substitute environment variables into templates")
    input = app.Arg("input", "Input file or stdin if not given").File()
    re = regexp.MustCompile(`\$(?:\{([A-Z][A-Z0-9_]*)\}|([A-Z][A-Z0-9_]*)\b)`)
)

func template(line string) string {
    var output = ""
    var nextIncludedIndex = 0
    matches := re.FindAllStringSubmatchIndex(line, -1)
    for i := 0; i < len(matches); i++ {
        match := matches[i]
        // find indices of range to replace in line plus the range that
        // gives the name of the variable
        replaceRange := match[0:2]
        var nameRange = match[2:4]
        if nameRange[0] == -1 {
            nameRange = match[4:6]
        }
        name := line[nameRange[0]:nameRange[1]]
        output += line[nextIncludedIndex:replaceRange[0]] + os.Getenv(name)
        nextIncludedIndex = replaceRange[1]
    }
    output += line[nextIncludedIndex:len(line)]
    return output
}

func main() {
    app.Version("0.0.1")
    app.Parse(os.Args[1:])

    var bio = bufio.NewReader(os.Stdin)
    for {
        var line, err = bio.ReadString('\n')
        if err != nil {
            if err != io.EOF {
                log.Fatal(err)
            }
            break
        }
        fmt.Printf("%s", template(line))
    }
}<commit_msg>Add possibility of using os.ExpandEnv<commit_after>package main

import (
    "bufio"
    "fmt"
    "io"
    "log"
    "os"
    "regexp"
    "gopkg.in/alecthomas/kingpin.v1"
)

var (
    app = kingpin.New("substenv", "Substitute environment variables into templates")
    expandFlag = app.Flag("expandenv", "Use os.ExpandEnv").Short('e').Bool()
    input = app.Arg("input", "Input file or stdin if not given").File()
    re = regexp.MustCompile(`\$(?:\{([A-Z][A-Z0-9_]*)\}|([A-Z][A-Z0-9_]*)\b)`)
)

func template(line string) string {
    var output = ""
    var nextIncludedIndex = 0
    matches := re.FindAllStringSubmatchIndex(line, -1)
    for i := 0; i < len(matches); i++ {
        match := matches[i]
        // find indices of range to replace in line plus the range that
        // gives the name of the variable
        replaceRange := match[0:2]
        var nameRange = match[2:4]
        if nameRange[0] == -1 {
            nameRange = match[4:6]
        }
        name := line[nameRange[0]:nameRange[1]]
        output += line[nextIncludedIndex:replaceRange[0]] + os.Getenv(name)
        nextIncludedIndex = replaceRange[1]
    }
    output += line[nextIncludedIndex:len(line)]
    return output
}

func main() {
    app.Version("0.0.1")
    app.Parse(os.Args[1:])

    var bio = bufio.NewReader(os.Stdin)
    for {
        var line, err = bio.ReadString('\n')
        if err != nil {
            if err != io.EOF {
                log.Fatal(err)
            }
            break
        }
        var out = ""
        if *expandFlag {
            out = os.ExpandEnv(line)
        } else {
            out = template(line)
        }
        fmt.Printf("%s", out)
    }
}
{'fraction_non_alphanumeric': 0.1520281060364101, 'fraction_numerical': 0.015330565314595975, 'mean_word_length': 2.0979228486646884, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '167577', 'n_tokens_mistral': 1080, 'n_tokens_neox': 1061, 'n_words': 237}