<commit_before>// Copyright 2016 <NAME>

// Colorout is a command line utility that colors command
// output. Each command is executed concurrently using bash
// shell and annotated with a different color.
package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
)

const shell = "bash"

const (
	colorReset   = "\x1b[0m"
	colorRed     = "\033[31m"
	colorGreen   = "\033[32m"
	colorYellow  = "\033[33m"
	colorBlue    = "\033[34m"
	colorMagenta = "\033[35m"
	colorCyan    = "\033[36m"
	colorWhite   = "\033[37m"
)

var colors = []string{
	colorRed,
	colorGreen,
	colorYellow,
	colorBlue,
	colorMagenta,
	colorCyan,
	colorWhite}

var mu sync.Mutex

func main() {
	log.SetFlags(0)
	var wg sync.WaitGroup
	args := os.Args[1:]
	if len(args) > len(colors) {
		log.Fatal("Too many commands!")
	}
	for i, command := range args {
		wg.Add(1)
		go func(i int, command string) {
			defer wg.Done()
			runCommand(i, command)
		}(i, command)
	}
	wg.Wait()
}

func runCommand(i int, command string) {
	out(os.Stdout, i, fmt.Sprint("Running: ", command))
	cmd := exec.Command(shell, "-c", command)

	// Close standard input
	stdin, err := cmd.StdinPipe()
	if err != nil {
		log.Fatal(err)
	}
	stdin.Close()

	// Read standard output and error
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}

	err = cmd.Start()
	if err != nil {
		log.Fatal(err)
	}

	go readOutput(os.Stdout, i, stdout)
	go readOutput(os.Stderr, i, stderr)

	err = cmd.Wait()
	if err != nil {
		log.Fatalf("%d Command finished with error %v", i, err)
	}
	out(os.Stdout, i, "Command exited successfully")
}

func readOutput(file *os.File, i int, r io.Reader) {
	br := bufio.NewReader(r)
	for {
		line, err := br.ReadString('\n')
		if err != nil {
			break
		}
		out(file, i, strings.TrimRight(line, "\n"))
	}
}

func out(file *os.File, i int, line string) {
	if i > len(colors) {
		panic("Color index out of range")
	}

	mu.Lock()
	defer mu.Unlock()
	file.WriteString(fmt.Sprintf("%s%d> %s%s\n", colors[i], i, line, colorReset))
}<commit_msg>Rewrite colorout with more correctness<commit_after>// Copyright 2016-2017 <NAME>.

// Command colorout is a colors and multiplexes output from tasks.
// Each task (external command) is executed concurrently using
// command shell and output is colored with a unique color per task.
package main

import (
	"context"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"

	"github.com/fatih/color"
)

var colors = []*color.Color{
	color.New(color.FgHiRed),
	color.New(color.FgHiGreen),
	color.New(color.FgHiYellow),
	color.New(color.FgHiBlue),
	color.New(color.FgHiMagenta),
	color.New(color.FgHiCyan),
	color.New(color.FgHiWhite),
	color.New(color.FgRed, color.ReverseVideo),
	color.New(color.FgGreen, color.ReverseVideo),
	color.New(color.FgYellow, color.ReverseVideo),
	color.New(color.FgBlue, color.ReverseVideo),
	color.New(color.FgMagenta, color.ReverseVideo),
	color.New(color.FgCyan, color.ReverseVideo),
	color.New(color.FgWhite, color.ReverseVideo),
}

func main() {
	fail := flag.Bool("fail", false, "terminate if any task fails with error")
	flag.Parse()
	tasks := flag.Args()

	if len(tasks) > len(colors) {
		log.Fatal("Too many tasks!")
	}

	// safeWriter protects stdout and stderr for concurrent access
	stdout := &safeWriter{W: os.Stdout}
	stderr := &safeWriter{W: os.Stderr}

	wg := &sync.WaitGroup{}
	wg.Add(len(tasks))
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	for i, command := range tasks {
		fmt.Fprintf(stderr, "%d Running: %s\n", i, command)

		go func(i int, command string) {
			defer wg.Done()
			if err := runCommand(ctx, i, command, stdout, stderr); err != nil {
				fmt.Fprintf(stderr, "command failed with %v\n", err)
				if *fail { // terminate other tasks on failure
					cancel()
				}
			}
		}(i, command)
	}
	wg.Wait()
}

func runCommand(ctx context.Context, i int, command string, stdout io.Writer, stderr io.Writer) error {
	commandLine := append(shellCommand(), command)
	cmd := exec.CommandContext(ctx, commandLine[0], commandLine[1:]...)

	cmd.Stdout = colorize(stdout, i)
	cmd.Stderr = colorize(stderr, i)
	defer cmd.Stdout.(io.Closer).Close()
	defer cmd.Stderr.(io.Closer).Close()

	if err := cmd.Start(); err != nil {
		return err
	}

	if err := cmd.Wait(); err != nil {
		return err
	}
	return nil
}

func shellCommand() []string {
	return []string{"bash", "-c"}
}

func colorize(dst io.Writer, i int) io.Writer {
	return &colorizer{
		dst: dst,
		i:   i,
	}
}

type colorizer struct {
	dst  io.Writer
	i    int
	prev string

	mu sync.Mutex
}

func (c *colorizer) Write(data []byte) (n int, err error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	lines := strings.Split(c.prev+string(data), "\n")
	n = 0
	for _, line := range lines[:len(lines)-1] {
		n += len(line) + 1
		_, err = colors[c.i].Fprintf(c.dst, "%d %v\n", c.i, line)
		if err != nil {
			return n, err
		}
	}
	c.prev = ""
	if n < len(data) {
		c.prev = string(data[n:])
	}
	n = len(data)
	return
}

func (c *colorizer) Close() error {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.prev != "" {
		_, err := colors[c.i].Fprintf(c.dst, "%d %v\n", c.i, c.prev)
		return err
	}
	return nil
}

type safeWriter struct {
	W io.Writer

	mu sync.Mutex
}

func (s *safeWriter) Write(data []byte) (n int, err error) {
	s.mu.Lock()
	n, err = s.W.Write(data)
	s.mu.Unlock()
	return
}
{'fraction_non_alphanumeric': 0.16433050534857985, 'fraction_numerical': 0.01051272593139063, 'mean_word_length': 3.5609756097560976, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4061514', 'n_tokens_mistral': 2262, 'n_tokens_neox': 2124, 'n_words': 547}