<commit_before>package system

import (
	"io"
	"os/exec"

	"github.com/rs/zerolog"
)

type zerologWriter struct {
	targetLog *zerolog.Logger
}

func (zw *zerologWriter) Write(p []byte) (n int, err error) {
	zw.targetLog.Info().Msg(string(p))
	return len(p), nil
}

// RunOSCommand properly executes a system command
// and writes the output to the provided logger
func RunOSCommand(cmd *exec.Cmd, logger *zerolog.Logger) error {
	logger.Debug().
		Interface("Arguments", cmd.Args).
		Str("Dir", cmd.Dir).
		Str("Path", cmd.Path).
		Interface("Env", cmd.Env).
		Msg("Running Command")

	outputWriter := &zerologWriter{targetLog: logger}
	cmdErr := RunAndCaptureOSCommand(cmd,
		outputWriter,
		outputWriter,
		logger)
	return cmdErr
}

// RunAndCaptureOSCommand runs the given command and
// captures the stdout and stderr
func RunAndCaptureOSCommand(cmd *exec.Cmd,
	stdoutWriter io.Writer,
	stderrWriter io.Writer,
	logger *zerolog.Logger) error {
	logger.Debug().
		Interface("Arguments", cmd.Args).
		Str("Dir", cmd.Dir).
		Str("Path", cmd.Path).
		Interface("Env", cmd.Env).
		Msg("Running Command")
	cmd.Stdout = stdoutWriter
	cmd.Stderr = stderrWriter
	return cmd.Run()

}<commit_msg>Proper logging for zerolog cmd output<commit_after>package system

import (
	"bufio"
	"bytes"
	"io"
	"os/exec"
	"strings"

	"github.com/rs/zerolog"
)

type zerologWriter struct {
	targetLog *zerolog.Logger
}

func (zw *zerologWriter) Write(p []byte) (n int, err error) {
	zw.targetLog.Info().Msg(string(p))
	return len(p), nil
}

// RunOSCommand properly executes a system command
// and writes the output to the provided logger
func RunOSCommand(cmd *exec.Cmd, logger *zerolog.Logger) error {
	logger.Debug().
		Interface("Arguments", cmd.Args).
		Str("Dir", cmd.Dir).
		Str("Path", cmd.Path).
		Interface("Env", cmd.Env).
		Msg("Running Command")

	outputWriter := &zerologWriter{targetLog: logger}
	cmdErr := RunAndCaptureOSCommand(cmd,
		outputWriter,
		outputWriter,
		logger)
	return cmdErr
}

// RunAndCaptureOSCommand runs the given command and
// captures the stdout and stderr
func RunAndCaptureOSCommand(cmd *exec.Cmd,
	stdoutWriter io.Writer,
	stderrWriter io.Writer,
	logger *zerolog.Logger) error {
	logger.Debug().
		Interface("Arguments", cmd.Args).
		Str("Dir", cmd.Dir).
		Str("Path", cmd.Path).
		Interface("Env", cmd.Env).
		Msg("Running Command")

	// Write the command to a buffer, split the lines, log them...
	var commandStdOutput bytes.Buffer
	var commandStdErr bytes.Buffer

	cmd.Stdout = &commandStdOutput
	cmd.Stderr = &commandStdErr
	cmdErr := cmd.Run()

	// Output each one...
	scannerStdout := bufio.NewScanner(&commandStdOutput)
	for scannerStdout.Scan() {
		text := strings.TrimSpace(scannerStdout.Text())
		if len(text) != 0 {
			logger.Info().Msg(text)
		}
	}
	scannerStderr := bufio.NewScanner(&commandStdErr)
	for scannerStderr.Scan() {
		text := strings.TrimSpace(scannerStderr.Text())
		if len(text) != 0 {
			logger.Error().Msg(text)
		}
	}
	return cmdErr
}
{'fraction_non_alphanumeric': 0.13931681178834562, 'fraction_numerical': 0.0006697923643670462, 'mean_word_length': 4.353046594982079, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1015030', 'n_tokens_mistral': 1066, 'n_tokens_neox': 1011, 'n_words': 256}