<commit_before>package main

import (
	//"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

var (
	DIFF     = ""
	CM_DIFF  = ""
	DIFFSTAT = ""
)

// Diff two files.  If forceNormal is True, custom diffs configured in the
// the environ are ignored for the regular `diff` command.
func diff(src, dest string, forceNormal ...bool) (*exec.Cmd, error) {
	var err error

	if len(DIFF) == 0 {
		CM_DIFF = os.Getenv("CM_DIFF")
		DIFF, _ = exec.LookPath("diff")
		if len(CM_DIFF) == 0 {
			CM_DIFF, err = exec.LookPath("colordiff")
			if err != nil {
				CM_DIFF = DIFF
			}
		}
	}

	if len(CM_DIFF) == 0 {
		return nil, errors.New("Could not find suitable diff executable in your PATH.")
	}

	if len(forceNormal) > 0 && forceNormal[0] {
		return exec.Command(DIFF, "-u", src, dest), nil
	}
	return exec.Command(CM_DIFF, "-u", src, dest), nil
}

// Runs diffstat on some diff output.  Returns diffstat command.
func diffstat(output string) (*exec.Cmd, error) {
	var err error
	var cmd *exec.Cmd

	if len(DIFFSTAT) == 0 {
		DIFFSTAT, _ = exec.LookPath("diffstat")
		if len(DIFFSTAT) == 0 {
			return nil, errors.New("Could not find suitable diffstat executable in your PATH.")
		}
	}

	cmd = exec.Command(DIFFSTAT, "-C")
	if err != nil {
		return nil, err
	}

	cmd.Stdin = strings.NewReader(output)
	if err != nil {
		return nil, err
	}
	return cmd, nil
}

// turn a path into a list of files.  if the path denotes a file, the list
// contains only that file.  if it contains a directory, a list of that
// directory is returned.  all paths are absolute.
func pathToFiles(path string) ([]string, error) {
	files := []string{}
	if isFile(path) {
		files = append(files, path)
	} else {
		stat, err := os.Stat(path)
		if err != nil {
			return files, err<commit_msg>darwin diffstat does not have -C option (what a surprise)<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

var (
	DIFF     = ""
	CM_DIFF  = ""
	DIFFSTAT = ""
)

// Diff two files.  If forceNormal is True, custom diffs configured in the
// the environ are ignored for the regular `diff` command.
func diff(src, dest string, forceNormal ...bool) (*exec.Cmd, error) {
	var err error

	if len(DIFF) == 0 {
		CM_DIFF = os.Getenv("CM_DIFF")
		DIFF, _ = exec.LookPath("diff")
		if len(CM_DIFF) == 0 {
			CM_DIFF, err = exec.LookPath("colordiff")
			if err != nil {
				CM_DIFF = DIFF
			}
		}
	}

	if len(CM_DIFF) == 0 {
		return nil, errors.New("Could not find suitable diff executable in your PATH.")
	}

	if len(forceNormal) > 0 && forceNormal[0] {
		return exec.Command(DIFF, "-u", src, dest), nil
	}
	return exec.Command(CM_DIFF, "-u", src, dest), nil
}

// Runs diffstat on some diff output.  Returns diffstat command.
func diffstat(output string) (*exec.Cmd, error) {
	var err error
	var cmd *exec.Cmd

	if len(DIFFSTAT) == 0 {
		DIFFSTAT, _ = exec.LookPath("diffstat")
		if len(DIFFSTAT) == 0 {
			return nil, errors.New("Could not find suitable diffstat executable in your PATH.")
		}
	}

	if runtime.GOOS == "darwin" {
		cmd = exec.Command(DIFFSTAT)
	} else {
		cmd = exec.Command(DIFFSTAT, "-C")
	}
	if err != nil {
		return nil, err
	}

	cmd.Stdin = strings.NewReader(output)
	if err != nil {
		return nil, err
	}
	return cmd, nil
}

// turn a path into a list of files.  if the path denotes a file, the list
// contains only that file.  if it contains a directory, a list of that
// directory is returned.  all paths are absolute.
func pathToFiles(path string) ([]string, error) {
	files := []string{}
	if isFile(path) {
		files = append(files, path)
	} else {
		stat, err := os.Stat(path)
		if err != nil {
			return files, err
{'fraction_non_alphanumeric': 0.13837837837837838, 'fraction_numerical': 0.0037837837837837837, 'mean_word_length': 3.2009080590238366, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5949584', 'n_tokens_mistral': 1445, 'n_tokens_neox': 1357, 'n_words': 438}