<commit_before>package core

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"syscall"

	"golang.org/x/text/encoding"
	"golang.org/x/text/transform"
)

// Cmd is command infomation.
type Cmd struct {
	Cmd     *exec.Cmd
	CmdLine string

	Stdin  bytes.Buffer
	Stdout bytes.Buffer
	Stderr bytes.Buffer

	ExitError error
	ExitCode  int

	StdinPipe  io.WriteCloser
	StdoutPipe io.ReadCloser
	StderrPipe io.ReadCloser

	StdinEnc  *encoding.Decoder
	StdoutEnc *encoding.Decoder
	StderrEnc *encoding.Decoder

	StdoutPrint bool
	StderrPrint bool

	wg sync.WaitGroup
}

// IsMatchStrs is whether str match or not.
func IsMatchStrs(str string, regStrs []string) (bool, error) {

	var err error

	if len(regStrs) == 0 {
		return true, nil
	}
	re, err := CompileStrs(regStrs)
	if err != nil {
		return false, err
	}
	return re.MatchString(str), nil
}

// CompileStrs is regexp strings compile to *regexp.Regexp.
func CompileStrs(regStrs []string) (*regexp.Regexp, error) {
	if len(regStrs) == 0 {
		return nil, fmt.Errorf("regStrs must be greater than or equal to 1")
	}
	regStr := "(" + strings.Join(regStrs, "|") + ")"
	re, err := regexp.Compile(regStr)
	if err != nil {
		return nil, err
	}
	return re, nil
}

// ScanPrintStdout is scan and print to stdout.
func ScanPrintStdout(scanner *bufio.Scanner, print bool) {
	for scanner.Scan() {
		if print {
			fmt.Fprintf(os.Stdout, "%s\n", scanner.Text())
		}
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

// ScanPrintStderr is scan and print to stderr.
func ScanPrintStderr(scanner *bufio.Scanner, print bool) {
	for scanner.Scan() {
		if print {
			fmt.Fprintf(os.Stderr, "%s\n", scanner.Text())
		}
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

// ReadToBuf is scan and store buffer.
func ReadToBuf(scanner *bufio.Scanner) (*bytes.Buffer, error) {
	buf := new(bytes.Buffer)
	for scanner.Scan() {
		buf.WriteString(scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return buf, nil
}

// BaseName is get file name without extension.
func BaseName(path string) string {
	base := filepath.Base(path)
	ext := filepath.Ext(path)

	re := regexp.MustCompile(ext + "$")
	return re.ReplaceAllString(base, "")
}

// GetCmdPath returns cmd abs path.
func GetCmdPath(cmd string) (string, error) {
	if filepath.IsAbs(cmd) {
		return cmd, nil
	}
	return exec.LookPath(cmd)
}

// FailOnError is fail if err occured.
func FailOnError(err error) {
	if err != nil {
		log.Fatalln(err)
	}
}

// GetGlobArgs return glob files.
func GetGlobArgs(args []string) ([]string, error) {

	var a []string

	for _, v := range args {
		files, err := filepath.Glob(v)
		if err != nil {
			return nil, err
		}
		a = append(a, files...)
	}

	return a, nil
}

// CmdStart start cmdFile.
func (c *Cmd) CmdStart() error {

	var err error

	c.StdoutPipe, err = c.Cmd.StdoutPipe()
	if err != nil {
		return err
	}

	c.StderrPipe, err = c.Cmd.StderrPipe()
	if err != nil {
		return err
	}

	c.StdinPipe, err = c.Cmd.StdinPipe()
	if err != nil {
		return err
	}

	if c.StdoutEnc == nil {
		c.wg.Add(1)
		go func() {
			defer c.wg.Done()
			ScanPrintStdout(bufio.NewScanner(io.TeeReader(c.StdoutPipe, &c.Stdout)), c.StdoutPrint)
		}()
	} else {
		c.wg.Add(1)
		go func() {
			defer c.wg.Done()
			ScanPrintStdout(bufio.NewScanner(transform.NewReader(io.TeeReader(c.StdoutPipe, &c.Stdout), c.StdoutEnc)), c.StdoutPrint)
		}()
	}

	if c.StderrEnc == nil {
		c.wg.Add(1)
		go func() {
			defer c.wg.Done()
			ScanPrintStderr(bufio.NewScanner(io.TeeReader(c.StderrPipe, &c.Stderr)), c.StderrPrint)
		}()
	} else {
		c.wg.Add(1)
		go func() {
			defer c.wg.Done()
			ScanPrintStderr(bufio.NewScanner(transform.NewReader(io.TeeReader(c.StderrPipe, &c.Stderr), c.StderrEnc)), c.StderrPrint)
		}()
	}

	err = c.Cmd.Start()
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>:bug: Fix ScanPrint arg order<commit_after>package core

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"syscall"

	"golang.org/x/text/encoding"
	"golang.org/x/text/transform"
)

// Cmd is command infomation.
type Cmd struct {
	Cmd     *exec.Cmd
	CmdLine string

	Stdin  bytes.Buffer
	Stdout bytes.Buffer
	Stderr bytes.Buffer

	ExitError error
	ExitCode  int

	StdinPipe  io.WriteCloser
	StdoutPipe io.ReadCloser
	StderrPipe io.ReadCloser

	StdinEnc  *encoding.Decoder
	StdoutEnc *encoding.Decoder
	StderrEnc *encoding.Decoder

	StdoutPrint bool
	StderrPrint bool

	wg sync.WaitGroup
}

// IsMatchStrs is whether str match or not.
func IsMatchStrs(str string, regStrs []string) (bool, error) {

	var err error

	if len(regStrs) == 0 {
		return true, nil
	}
	re, err := CompileStrs(regStrs)
	if err != nil {
		return false, err
	}
	return re.MatchString(str), nil
}

// CompileStrs is regexp strings compile to *regexp.Regexp.
func CompileStrs(regStrs []string) (*regexp.Regexp, error) {
	if len(regStrs) == 0 {
		return nil, fmt.Errorf("regStrs must be greater than or equal to 1")
	}
	regStr := "(" + strings.Join(regStrs, "|") + ")"
	re, err := regexp.Compile(regStr)
	if err != nil {
		return nil, err
	}
	return re, nil
}

// ScanPrintStdout is scan and print to stdout.
func ScanPrintStdout(scanner *bufio.Scanner, print bool) {
	for scanner.Scan() {
		if print {
			fmt.Fprintf(os.Stdout, "%s\n", scanner.Text())
		}
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

// ScanPrintStderr is scan and print to stderr.
func ScanPrintStderr(scanner *bufio.Scanner, print bool) {
	for scanner.Scan() {
		if print {
			fmt.Fprintf(os.Stderr, "%s\n", scanner.Text())
		}
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

// ReadToBuf is scan and store buffer.
func ReadToBuf(scanner *bufio.Scanner) (*bytes.Buffer, error) {
	buf := new(bytes.Buffer)
	for scanner.Scan() {
		buf.WriteString(scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return buf, nil
}

// BaseName is get file name without extension.
func BaseName(path string) string {
	base := filepath.Base(path)
	ext := filepath.Ext(path)

	re := regexp.MustCompile(ext + "$")
	return re.ReplaceAllString(base, "")
}

// GetCmdPath returns cmd abs path.
func GetCmdPath(cmd string) (string, error) {
	if filepath.IsAbs(cmd) {
		return cmd, nil
	}
	return exec.LookPath(cmd)
}

// FailOnError is fail if err occured.
func FailOnError(err error) {
	if err != nil {
		log.Fatalf("%+v\n", err)
	}
}

// GetGlobArgs return glob files.
func GetGlobArgs(args []string) ([]string, error) {

	var a []string

	for _, v := range args {
		files, err := filepath.Glob(v)
		if err != nil {
			return nil, err
		}
		a = append(a, files...)
	}

	return a, nil
}

// CmdStart start cmdFile.
func (c *Cmd) CmdStart() error {

	var err error

	c.StdoutPipe, err = c.Cmd.StdoutPipe()
	if err != nil {
		return err
	}

	c.StderrPipe, err = c.Cmd.StderrPipe()
	if err != nil {
		return err
	}

	c.StdinPipe, err = c.Cmd.StdinPipe()
	if err != nil {
		return err
	}

	if c.StdoutEnc == nil {
		c.wg.Add(1)
		go func() {
			defer c.wg.Done()
			ScanPrintStdout(bufio.NewScanner(io.TeeReader(c.StdoutPipe, &c.Stdout)), c.StdoutPrint)
		}()
	} else {
		c.wg.Add(1)
		go func() {
			defer c.wg.Done()
			ScanPrintStdout(bufio.NewScanner(io.TeeReader(transform.NewReader(c.StdoutPipe, c.StdoutEnc), &c.Stdout)), c.StdoutPrint)
		}()
	}

	if c.StderrEnc == nil {
		c.wg.Add(1)
		go func() {
			defer c.wg.Done()
			ScanPrintStderr(bufio.NewScanner(io.TeeReader(c.StderrPipe, &c.Stderr)), c.StderrPrint)
		}()
	} else {
		c.wg.Add(1)
		go func() {
			defer c.wg.Done()
			ScanPrintStderr(bufio.NewScanner(io.TeeReader(transform.NewReader(c.StderrPipe, c.StderrEnc), &c.Stderr)), c.StderrPrint)
		}()
	}

	err = c.Cmd.Start()
	if err != nil {
		return err
	}

	return nil
}

{'fraction_non_alphanumeric': 0.14789446981227802, 'fraction_numerical': 0.0017757483510908167, 'mean_word_length': 3.412423055400112, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '541509', 'n_tokens_mistral': 3381, 'n_tokens_neox': 3166, 'n_words': 757}