<commit_before>package get

import (
	"bufio"
	"io"
	"os"
	"path"
)

// Storage allows to store data in a local directory
type Storage struct {
	directory string
}

// NewStorage returns a new Storage given a local directory
func NewStorage(directory string) *Storage {
	return &Storage{directory}
}

// NewStoringReader returns a reader that will also store any read data to filename
func (s *Storage) NewStoringReader(filename string, reader io.ReadCloser) (result io.ReadCloser, err error) {
	fullPath := path.Join(s.directory, filename)
	// attempt to create any missing directories in the full path
	err = os.MkdirAll(path.Dir(fullPath), os.ModePerm)
	if err != nil {
		return
	}

	file, err := os.Create(fullPath)
	if err != nil {
		return
	}

	writer := bufio.NewWriter(file)
	teeReader := io.TeeReader(reader, writer)

	result = &storingReader{reader, writer, teeReader}
	return
}

// storingReader uses a TeeReader to copy data from a reader to a writer
type storingReader struct {
	reader    io.ReadCloser
	writer    *bufio.Writer
	teeReader io.Reader
}

// Read delegates to the TeeReader implementation
func (t *storingReader) Read(p []byte) (n int, err error) {
	return t.teeReader.Read(p)
}

// Closes the internal reader and flushes the writer
func (t *storingReader) Close() (err error) {
	err = t.reader.Close()
	if err != nil {
		return
	}
	return t.writer.Flush()
}
<commit_msg>Bugfix: make sure Reader is always read fully<commit_after>package get

import (
	"bufio"
	"io"
	"io/ioutil"
	"os"
	"path"
)

// Storage allows to store data in a local directory
type Storage struct {
	directory string
}

// NewStorage returns a new Storage given a local directory
func NewStorage(directory string) *Storage {
	return &Storage{directory}
}

// NewStoringReader returns a reader that will also store any read data to filename
func (s *Storage) NewStoringReader(filename string, reader io.ReadCloser) (result io.ReadCloser, err error) {
	fullPath := path.Join(s.directory, filename)
	// attempt to create any missing directories in the full path
	err = os.MkdirAll(path.Dir(fullPath), os.ModePerm)
	if err != nil {
		return
	}

	file, err := os.Create(fullPath)
	if err != nil {
		return
	}

	writer := bufio.NewWriter(file)
	teeReader := io.TeeReader(reader, writer)

	result = &storingReader{reader, writer, teeReader}
	return
}

// storingReader uses a TeeReader to copy data from a reader to a writer
type storingReader struct {
	reader    io.ReadCloser
	writer    *bufio.Writer
	teeReader io.Reader
}

// Read delegates to the TeeReader implementation
func (t *storingReader) Read(p []byte) (n int, err error) {
	return t.teeReader.Read(p)
}

// Closes the internal reader and flushes the writer
func (t *storingReader) Close() (err error) {
	ioutil.ReadAll(t.teeReader)
	err = t.reader.Close()
	if err != nil {
		return
	}
	return t.writer.Flush()
}

{'fraction_non_alphanumeric': 0.10111576011157601, 'fraction_numerical': 0.0, 'mean_word_length': 4.059964726631393, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2042594', 'n_tokens_mistral': 957, 'n_tokens_neox': 932, 'n_words': 325}