<commit_before>// +build linux darwin freebsd

package commands

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
)

var (
	fileSuffix = "kvexpress"
)

// ReadFile reads a file in the filesystem and returns a string.
func ReadFile(filepath string) string {
	dat, err := ioutil.ReadFile(filepath)
	if err != nil {
		dat = []byte("")
	}
	return string(dat)
}

// SortFile takes a string, splits it into lines, removes all blank lines using
// BlankLineStrip() and then sorts the remaining lines.
func SortFile(file string) string {
	Log("sorting='true'", "debug")
	lines := strings.Split(file, "\n")
	lines = BlankLineStrip(lines)
	sort.Strings(lines)
	return strings.Join(lines, "\n")
}

// BlankLineStrip takes a slice of strings, ranges over them and only returns
// a slice of strings where the lines weren't blank.
func BlankLineStrip(data []string) []string {
	Log(fmt.Sprintf("in: stripping_blank_lines='true'"), "debug")
	var stripped []string
	for _, str := range data {
		if str != "" {
			stripped = append(stripped, str)
		}
	}
	return stripped
}

// CheckFullPath will check the path and recursively create directories if they don't
// exist.
func CheckFullPath(file string) {
	targetDirectory := path.Dir(file)
	// If there is a file with the same name in the targetDirectory path - it will error.
	// It will not overwrite it.
	err := os.MkdirAll(targetDirectory, os.FileMode(0755))
	if err != nil {
		Log(fmt.Sprintf("function='CheckFullPath' panic='true' file='%s'", targetDirectory), "info")
		fmt.Printf("Panic: Could not create directories: '%s'\n", targetDirectory)
		StatsdPanic(targetDirectory, "create_directory")
	}
}

// WriteFile writes a string to a filepath. It also chowns the file to the owner and group
// of the user running the program if it's not set as a different user.
func WriteFile(data string, filepath string, perms int, owner string) {
	tmpFile := fmt.Sprintf("%s.%s", filepath, fileSuffix)
	// If a directory doesn't exist then that's a bad thing.
	// Caused some problems with Consul and file descriptors after a long weekend erroring.
	CheckFullPath(filepath)
	// Write the file to a different name.
	err := ioutil.WriteFile(tmpFile, []byte(data), os.FileMode(perms))
	if err != nil {
		Log(fmt.Sprintf("function='WriteFile' panic='true' file='%s'", filepath), "info")
		fmt.Printf("Panic: Could not write file: '%s'\n", filepath)
		StatsdPanic(filepath, "write_file")
	}
	// Rename the file so it's not truncated for 1 microsecond
	// which is actually important at high velocities.
	err = os.Rename(tmpFile, filepath)
	if err != nil {
		Log(fmt.Sprintf("function='Rename' panic='true' file='%s'", filepath), "info")
		fmt.Printf("Panic: Could not rename file: '%s'\n", filepath)
		StatsdPanic(filepath, "rename_file")
	}
	// Chown the file.
	fileChown, oid, gid := ChownFile(filepath, owner)
	Log(fmt.Sprintf("file_wrote='true' location='%s' permissions='%s'", filepath, strconv.FormatInt(int64(perms), 8)), "debug")
	Log(fmt.Sprintf("file_chown='%t' location='%s' owner='%d' group='%d'", fileChown, filepath, oid, gid), "debug")
}
<commit_msg>Be a little more consistent so it makes more sense.<commit_after>// +build linux darwin freebsd

package commands

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
)

var (
	fileSuffix = "kvexpress"
)

// ReadFile reads a file in the filesystem and returns a string.
func ReadFile(filepath string) string {
	dat, err := ioutil.ReadFile(filepath)
	if err != nil {
		dat = []byte("")
	}
	return string(dat)
}

// SortFile takes a string, splits it into lines, removes all blank lines using
// BlankLineStrip() and then sorts the remaining lines.
func SortFile(file string) string {
	Log("sorting='true'", "debug")
	lines := strings.Split(file, "\n")
	lines = BlankLineStrip(lines)
	sort.Strings(lines)
	return strings.Join(lines, "\n")
}

// BlankLineStrip takes a slice of strings, ranges over them and only returns
// a slice of strings where the lines weren't blank.
func BlankLineStrip(data []string) []string {
	Log(fmt.Sprintf("in: stripping_blank_lines='true'"), "debug")
	var stripped []string
	for _, str := range data {
		if str != "" {
			stripped = append(stripped, str)
		}
	}
	return stripped
}

// CheckFullPath will check the path and recursively create directories if they don't
// exist.
func CheckFullPath(file string) {
	targetDirectory := path.Dir(file)
	// If there is a file with the same name in the targetDirectory path - it will error.
	// It will not overwrite it.
	err := os.MkdirAll(targetDirectory, os.FileMode(0755))
	if err != nil {
		Log(fmt.Sprintf("function='CheckFullPath' panic='true' file='%s'", targetDirectory), "info")
		fmt.Printf("Panic: Could not create directories: '%s'\n", targetDirectory)
		StatsdPanic(targetDirectory, "create_directory")
	}
}

// WriteFile writes a string to a filepath. It also chowns the file to the owner and group
// of the user running the program if it's not set as a different user.
func WriteFile(data string, filepath string, perms int, owner string) {
	tmpFilepath := fmt.Sprintf("%s.%s", filepath, fileSuffix)
	// If a directory doesn't exist then that's a bad thing.
	// Caused some problems with Consul and file descriptors after a long weekend erroring.
	CheckFullPath(filepath)
	// Write the file to the tmpFilepath.
	err := ioutil.WriteFile(tmpFilepath, []byte(data), os.FileMode(perms))
	if err != nil {
		Log(fmt.Sprintf("function='WriteFile' panic='true' file='%s'", filepath), "info")
		fmt.Printf("Panic: Could not write file: '%s'\n", filepath)
		StatsdPanic(filepath, "write_file")
	}
	// Rename the file so it's not truncated for 1 microsecond
	// which is actually important at high velocities.
	err = os.Rename(tmpFilepath, filepath)
	if err != nil {
		Log(fmt.Sprintf("function='Rename' panic='true' file='%s'", filepath), "info")
		fmt.Printf("Panic: Could not rename file: '%s'\n", filepath)
		StatsdPanic(filepath, "rename_file")
	}
	// Chown the file.
	fileChown, oid, gid := ChownFile(filepath, owner)
	Log(fmt.Sprintf("file_wrote='true' location='%s' permissions='%s'", filepath, strconv.FormatInt(int64(perms), 8)), "debug")
	Log(fmt.Sprintf("file_chown='%t' location='%s' owner='%d' group='%d'", fileChown, filepath, oid, gid), "debug")
}

{'fraction_non_alphanumeric': 0.13876055440497054, 'fraction_numerical': 0.0025489883702405607, 'mean_word_length': 4.575488454706927, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '184146', 'n_tokens_mistral': 2146, 'n_tokens_neox': 2081, 'n_words': 680}