<commit_before>package runner

import (
	"os"
	"os/exec"

	st "github.com/dyweb/Ayi/common/structure"
	"github.com/kballard/go-shellquote"
	"github.com/pkg/errors"
)

// commands that should be called using shell,
// because mostly they would be expecting shell expansion on parameters
var shellCommands = st.NewSet("rm", "cp", "mv", "mkdir", "tar")

// NewCmd can properly split the executable with its arguments
// TODO: may need to add a context to handle things like using shell or not
func NewCmd(cmdStr string) (*exec.Cmd, error) {
	segments, err := shellquote.Split(cmdStr)
	if err != nil {
		return nil, errors.Wrap(err, "can't parse command")
	}
	return exec.Command(segments[0], segments[1:]...), nil
}<commit_msg>[common][runner] Add echo to shell list<commit_after>package runner

import (
	"os"
	"os/exec"

	st "github.com/dyweb/Ayi/common/structure"
	"github.com/kballard/go-shellquote"
	"github.com/pkg/errors"
)

// commands that should be called using shell,
// because mostly they would be expecting shell expansion on parameters
var shellCommands = st.NewSet("echo", "rm", "cp", "mv", "mkdir", "tar")

// NewCmd can properly split the executable with its arguments
// TODO: may need to add a context to handle things like using shell or not
func NewCmd(cmdStr string) (*exec.Cmd, error) {
	segments, err := shellquote.Split(cmdStr)
	if err != nil {
		return nil, errors.Wrap(err, "can't parse command")
	}
	return exec.Command(segments[0], segments[1:]...), nil
}
{'fraction_non_alphanumeric': 0.13427800269905532, 'fraction_numerical': 0.002699055330634278, 'mean_word_length': 4.392727272727273, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2231584', 'n_tokens_mistral': 489, 'n_tokens_neox': 474, 'n_words': 162}