<commit_before>package pointer

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"os"
)

// consistentFileWriter ensures that the file matching the given SHA-256
// signature is written to the given path.
type consistentFileWriter struct {
	Path    string
	Sha256  string
	writer  io.Writer
	file    *os.File
	tmpFile *os.File
	hasher  hash.Hash
}

func newFile(path, sha256Sig string) (*consistentFileWriter, error) {
	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
	if err != nil {
		return nil, err
	}

	tmpFile, err := ioutil.TempFile("", sha256Sig)
	if err != nil {
		return nil, err
	}

	h := sha256.New()
	w := io.MultiWriter(tmpFile, h)
	return &consistentFileWriter{path, sha256Sig, w, file, tmpFile, h}, nil
}

func (w *consistentFileWriter) Write(p []byte) (int, error) {
	return w.writer.Write(p)
}

func (w *consistentFileWriter) Close() error {
	defer func() {
		w.tmpFile.Close()
		os.RemoveAll(w.tmpFile.Name())<commit_msg>cleanup the opened media file<commit_after>package pointer

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"os"
)

// consistentFileWriter ensures that the file matching the given SHA-256
// signature is written to the given path.
type consistentFileWriter struct {
	Path    string
	Sha256  string
	writer  io.Writer
	file    *os.File
	tmpFile *os.File
	hasher  hash.Hash
}

func newFile(path, sha256Sig string) (*consistentFileWriter, error) {
	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
	if err != nil {
		return nil, err
	}

	tmpFile, err := ioutil.TempFile("", sha256Sig)
	if err != nil {
		return nil, err
	}

	h := sha256.New()
	w := io.MultiWriter(tmpFile, h)
	return &consistentFileWriter{path, sha256Sig, w, file, tmpFile, h}, nil
}

func (w *consistentFileWriter) Write(p []byte) (int, error) {
	return w.writer.Write(p)
}

func (w *consistentFileWriter) Close() error {
	defer func() {
		w.file.Close()
		w.tmpFile.Close()
		os.RemoveAll(w.tmpFile.Name())
{'fraction_non_alphanumeric': 0.1338621715418939, 'fraction_numerical': 0.02478929102627665, 'mean_word_length': 4.045, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4898429', 'n_tokens_mistral': 814, 'n_tokens_neox': 758, 'n_words': 202}