<commit_before>package main

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/google/shlex"
	kingpin "gopkg.in/alecthomas/kingpin.v3-unstable"
)

type Vars map[string]string

func (v Vars) Copy() Vars {
	out := Vars{}
	for k, v := range v {
		out[k] = v
	}
	return out
}

func (v Vars) Replace(s string) string {
	for k, v := range v {
		prefix := regexp.MustCompile(fmt.Sprintf("{%s=([^}]*)}", k))
		if v != "" {
			s = prefix.ReplaceAllString(s, "$1")
		} else {
			s = prefix.ReplaceAllString(s, "")
		}
		s = strings.Replace(s, fmt.Sprintf("{%s}", k), v, -1)
	}
	return s
}

type linterState struct {
	*Linter
	issues   chan *Issue
	vars     Vars
	exclude  *regexp.Regexp
	include  *regexp.Regexp
	deadline <-chan time.Time
}

func (l *linterState) Partitions(paths []string) ([][]string, error) {
	cmdArgs, err := parseCommand(l.command())
	if err != nil {
		return nil, err
	}
	parts, err := l.Linter.PartitionStrategy(cmdArgs, paths)
	if err != nil {
		return nil, err
	}
	return parts, nil
}

func (l *linterState) command() string {
	return l.vars.Replace(l.Command)
}

func runLinters(linters map[string]*Linter, paths []string, concurrency int, exclude, include *regexp.Regexp) (chan *Issue, chan error) {
	errch := make(chan error, len(linters))
	concurrencych := make(chan bool, concurrency)
	incomingIssues := make(chan *Issue, 1000000)

	directiveParser := newDirectiveParser()
	if config.WarnUnmatchedDirective {
		directiveParser.LoadFiles(paths)
	}

	processedIssues := maybeSortIssues(filterIssuesViaDirectives(
		directiveParser, maybeAggregateIssues(incomingIssues)))

	vars := Vars{
		"duplthreshold":    fmt.Sprintf("%d", config.DuplThreshold),
		"mincyclo":         fmt.Sprintf("%d", config.Cyclo),
		"maxlinelength":    fmt.Sprintf("%d", config.LineLength),
		"misspelllocale":   fmt.Sprintf("%s", config.MisspellLocale),
		"min_confidence":   fmt.Sprintf("%f", config.MinConfidence),
		"min_occurrences":  fmt.Sprintf("%d", config.MinOccurrences),
		"min_const_length": fmt.Sprintf("%d", config.MinConstLength),
		"tests":            "",
		"not_tests":        "true",
	}
	if config.Test {
		vars["tests"] = "true"
		vars["not_tests"] = ""
	}

	wg := &sync.WaitGroup{}
	id := 1
	for _, linter := range linters {
		deadline := time.After(config.Deadline.Duration())
		state := &linterState{
			Linter:   linter,
			issues:   incomingIssues,
			vars:     vars,
			exclude:  exclude,
			include:  include,
			deadline: deadline,
		}

		partitions, err := state.Partitions(paths)
		if err != nil {
			errch <- err
			continue
		}
		for _, args := range partitions {
			wg.Add(1)
			concurrencych <- true
			// Call the goroutine with a copy of the args array so that the
			// contents of the array are not modified by the next iteration of
			// the above for loop
			go func(id int, args []string) {
				err := executeLinter(id, state, args)
				if err != nil {
					errch <- err
				}
				<-concurrencych
				wg.Done()
			}(id, args)
			id++
		}
	}

	go func() {
		wg.Wait()
		close(incomingIssues)
		close(errch)
	}()
	return processedIssues, errch
}

func executeLinter(id int, state *linterState, args []string) error {
	if len(args) == 0 {
		return fmt.Errorf("missing linter command")
	}

	start := time.Now()
	dbg := namespacedDebug(fmt.Sprintf("[%s.%d]: ", state.Name, id))
	dbg("executing %s", strings.Join(args, " "))
	buf := bytes.NewBuffer(nil)
	command := args[0]
	cmd := exec.Command(command, args[1:]...) // nolint: gosec
	cmd.Stdout = buf
	cmd.Stderr = buf
	err := cmd.Start()
	if err != nil {
		return fmt.Errorf("failed to execute linter %s: %s", command, err)
	}

	done := make(chan bool)
	go func() {
		err = cmd.Wait()
		done <- true
	}()

	// Wait for process to complete or deadline to expire.
	select {
	case <-done:

	case <-state.deadline:
		err = fmt.Errorf("deadline exceeded by linter %s (try increasing --deadline)",
			state.Name)
		kerr := cmd.Process.Kill()
		if kerr != nil {
			warning("failed to kill %s: %s", state.Name, kerr)
		}
		return err
	}

	if err != nil {
		dbg("warning: %s returned %s: %s", command, err, buf.String())
	}
<commit_msg>Pass error out of goroutine via channel<commit_after>package main

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/google/shlex"
	kingpin "gopkg.in/alecthomas/kingpin.v3-unstable"
)

type Vars map[string]string

func (v Vars) Copy() Vars {
	out := Vars{}
	for k, v := range v {
		out[k] = v
	}
	return out
}

func (v Vars) Replace(s string) string {
	for k, v := range v {
		prefix := regexp.MustCompile(fmt.Sprintf("{%s=([^}]*)}", k))
		if v != "" {
			s = prefix.ReplaceAllString(s, "$1")
		} else {
			s = prefix.ReplaceAllString(s, "")
		}
		s = strings.Replace(s, fmt.Sprintf("{%s}", k), v, -1)
	}
	return s
}

type linterState struct {
	*Linter
	issues   chan *Issue
	vars     Vars
	exclude  *regexp.Regexp
	include  *regexp.Regexp
	deadline <-chan time.Time
}

func (l *linterState) Partitions(paths []string) ([][]string, error) {
	cmdArgs, err := parseCommand(l.command())
	if err != nil {
		return nil, err
	}
	parts, err := l.Linter.PartitionStrategy(cmdArgs, paths)
	if err != nil {
		return nil, err
	}
	return parts, nil
}

func (l *linterState) command() string {
	return l.vars.Replace(l.Command)
}

func runLinters(linters map[string]*Linter, paths []string, concurrency int, exclude, include *regexp.Regexp) (chan *Issue, chan error) {
	errch := make(chan error, len(linters))
	concurrencych := make(chan bool, concurrency)
	incomingIssues := make(chan *Issue, 1000000)

	directiveParser := newDirectiveParser()
	if config.WarnUnmatchedDirective {
		directiveParser.LoadFiles(paths)
	}

	processedIssues := maybeSortIssues(filterIssuesViaDirectives(
		directiveParser, maybeAggregateIssues(incomingIssues)))

	vars := Vars{
		"duplthreshold":    fmt.Sprintf("%d", config.DuplThreshold),
		"mincyclo":         fmt.Sprintf("%d", config.Cyclo),
		"maxlinelength":    fmt.Sprintf("%d", config.LineLength),
		"misspelllocale":   fmt.Sprintf("%s", config.MisspellLocale),
		"min_confidence":   fmt.Sprintf("%f", config.MinConfidence),
		"min_occurrences":  fmt.Sprintf("%d", config.MinOccurrences),
		"min_const_length": fmt.Sprintf("%d", config.MinConstLength),
		"tests":            "",
		"not_tests":        "true",
	}
	if config.Test {
		vars["tests"] = "true"
		vars["not_tests"] = ""
	}

	wg := &sync.WaitGroup{}
	id := 1
	for _, linter := range linters {
		deadline := time.After(config.Deadline.Duration())
		state := &linterState{
			Linter:   linter,
			issues:   incomingIssues,
			vars:     vars,
			exclude:  exclude,
			include:  include,
			deadline: deadline,
		}

		partitions, err := state.Partitions(paths)
		if err != nil {
			errch <- err
			continue
		}
		for _, args := range partitions {
			wg.Add(1)
			concurrencych <- true
			// Call the goroutine with a copy of the args array so that the
			// contents of the array are not modified by the next iteration of
			// the above for loop
			go func(id int, args []string) {
				err := executeLinter(id, state, args)
				if err != nil {
					errch <- err
				}
				<-concurrencych
				wg.Done()
			}(id, args)
			id++
		}
	}

	go func() {
		wg.Wait()
		close(incomingIssues)
		close(errch)
	}()
	return processedIssues, errch
}

func executeLinter(id int, state *linterState, args []string) error {
	if len(args) == 0 {
		return fmt.Errorf("missing linter command")
	}

	start := time.Now()
	dbg := namespacedDebug(fmt.Sprintf("[%s.%d]: ", state.Name, id))
	dbg("executing %s", strings.Join(args, " "))
	buf := bytes.NewBuffer(nil)
	command := args[0]
	cmd := exec.Command(command, args[1:]...) // nolint: gosec
	cmd.Stdout = buf
	cmd.Stderr = buf
	err := cmd.Start()
	if err != nil {
		return fmt.Errorf("failed to execute linter %s: %s", command, err)
	}

	done := make(chan error)
	go func() {
		done <- cmd.Wait()
	}()

	// Wait for process to complete or deadline to expire.
	select {
	case err = <-done:

	case <-state.deadline:
		err = fmt.Errorf("deadline exceeded by linter %s (try increasing --deadline)",
			state.Name)
		kerr := cmd.Process.Kill()
		if kerr != nil {
			warning("failed to kill %s: %s", state.Name, kerr)
		}
		return err
	}

	if err != nil {
		dbg("warning: %s returned %s: %s", command, err, buf.String())
	}

{'fraction_non_alphanumeric': 0.1580961492465917, 'fraction_numerical': 0.0035876584549150922, 'mean_word_length': 3.5205405405405403, 'pattern_counts': {'":': 18, '<': 19, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2421914', 'n_tokens_mistral': 3353, 'n_tokens_neox': 3045, 'n_words': 781}