<commit_before>package glob

import (
	"io"
	"os/exec"
	"path/filepath"
	"strconv"
)

// Runner contains information related to creating blobs.
type Runner struct {
	Src    io.Reader
	Dest   string
	Amount int
	Unit   string
}

func NewRunner(src io.Reader, dst, unit string, amnt int) *Runner {
	r := new(Runner)

	r.Src = src
	r.Dest = dst
	r.Amount = amnt
	r.Unit = unit

	return r
}

func (r *Runner) Mk() error {
	for i := 1; i <= r.Amount; i++ {
		fileName := strconv.Itoa(i) + r.Unit

		err := newCreateFile(r.Src, r.Dest, fileName, r.Unit)
		if err != nil {
			return err
		}
	}

	return nil
}

func newCreateFile(src io.Reader, dst, fileName, unit string) error {
	cmd := exec.Command("touch", filepath.Join(dst, fileName))

	err := cmd.Run()
	if err != nil {
		return err
	}

	return nil
}<commit_msg>blob content can now be filled with stdin.<commit_after>package glob

import (
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"sync"
)

var (
	once sync.Once
)

// Runner contains information related to creating blobs.
type Runner struct {
	Src    io.Reader
	Dest   string
	Amount int
	Unit   string
}

func NewRunner(src io.Reader, dst, unit string, amnt int) *Runner {
	r := new(Runner)

	r.Src = src
	r.Dest = dst
	r.Amount = amnt
	r.Unit = unit

	return r
}

func (r *Runner) Mk() error {
	srcContent, err := ioutil.ReadAll(r.Src)
	if err != nil {
		return err
	}

	for i := 1; i <= r.Amount; i++ {
		fileName := strconv.Itoa(i) + r.Unit

		err := CreateBlob(srcContent, r.Dest, fileName, r.Unit)
		if err != nil {
			return err
		}
	}

	return nil
}

func CreateBlob(content []byte, dst, fileName, unit string) error {
	fullBlobPath := filepath.Join(dst, fileName)

	// Create a new file.
	newFile, err := createBlobFile(fullBlobPath)
	if err != nil {
		return err
	}
	defer newFile.Close()

	// Fill it with junk.
	err = fillFile(newFile, content)
	if err != nil {
		return err
	}

	return nil
}

// should be the new create file, need to rename.
func createBlobFile(src string) (*os.File, error) {
	return os.Create(src)
}

func fillFile(file *os.File, content []byte) error {

	bytesWritten, err := file.Write(content)
	if err != nil {
		return err
	}

	log.Printf("%d bytes written to file.", bytesWritten)

	return err
}
{'fraction_non_alphanumeric': 0.13082039911308205, 'fraction_numerical': 0.0008869179600886918, 'mean_word_length': 3.162361623616236, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3888632', 'n_tokens_mistral': 884, 'n_tokens_neox': 852, 'n_words': 263}