<commit_before>package libkb

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
)

type Source interface {
	io.ReadCloser
	Open() error
}

type Sink interface {
	io.WriteCloser
	Open() error
	HitError(err error) error
}

type BufferSource struct {
	data string
	buf  *bytes.Buffer
}

func NewBufferSource(s string) *BufferSource {
	return &BufferSource{data: s}
}

func (b *BufferSource) Open() error {
	b.buf = bytes.NewBufferString(b.data)
	return nil
}

func (b *BufferSource) Read(p []byte) (n int, err error) {
	return b.buf.Read(p)
}

func (b *BufferSource) Close() error { return nil }

type StdinSource struct {
	open bool
}

func (b *StdinSource) Open() error {
	b.open = true
	return nil
}

func (b *StdinSource) Close() error {
	b.open = false
	return nil
}

func (b *StdinSource) Read(p []byte) (n int, err error) {
	if b.open {
		return os.Stdin.Read(p)
	} else {
		return 0, io.EOF
	}
}

type FileSource struct {
	name string
	file *os.File
}

func NewFileSource(s string) *FileSource {
	return &FileSource{name: s}
}

func (s *FileSource) Open() error {
	f, err := os.OpenFile(s.name, os.O_RDONLY, 0)
	if err != nil {
		return err
	}
	s.file = f
	return nil
}

func (s *FileSource) Close() error {
	if s.file != nil {
		err := s.file.Close()
		s.file = nil
		return err
	} else {
		return io.EOF
	}
}

func (s *FileSource) Read(p []byte) (n int, err error) {
	if s.file == nil {
		return 0, io.EOF
	} else {
		return s.file.Read(p)
	}
}

type StdoutSink struct {
	open bool
}

func (s *StdoutSink) Open() error {
	s.open = true
	return nil
}

func (s *StdoutSink) Close() error {
	if !s.open {
		return io.EOF
	}
	s.open = false
	return nil
}

func (s *StdoutSink) Write(b []byte) (n int, err error) {
	return os.Stdout.Write(b)
}

func (s *StdoutSink) HitError(e error) error { return nil }

type FileSink struct {
	name string
	file *os.File
	bufw *bufio.Writer
}

func NewFileSink(s string) *FileSink {
	return &FileSink{name: s}
}

func (s *FileSink) Open() error {
	f, err := os.OpenFile(s.name, os.O_WRONLY, UMASKABLE_PERM_FILE)
	if err != nil {
		return fmt.Errorf("Failed to open %s for writing: %s",
			s.name, err.Error())
	}
	s.file = f
	s.bufw = bufio.NewWriter(f)
	return nil
}

func (s *FileSink) Write(b []byte) (n int, err error) {
	if s.bufw == nil {
		return 0, io.EOF
	} else {
		return s.bufw.Write(b)
	}
}

func (s *FileSink) Close() error {
	s.bufw.Flush()
	e := s.file.Close()<commit_msg>small fix to writing files<commit_after>package libkb

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
)

type Source interface {
	io.ReadCloser
	Open() error
}

type Sink interface {
	io.WriteCloser
	Open() error
	HitError(err error) error
}

type BufferSource struct {
	data string
	buf  *bytes.Buffer
}

func NewBufferSource(s string) *BufferSource {
	return &BufferSource{data: s}
}

func (b *BufferSource) Open() error {
	b.buf = bytes.NewBufferString(b.data)
	return nil
}

func (b *BufferSource) Read(p []byte) (n int, err error) {
	return b.buf.Read(p)
}

func (b *BufferSource) Close() error { return nil }

type StdinSource struct {
	open bool
}

func (b *StdinSource) Open() error {
	b.open = true
	return nil
}

func (b *StdinSource) Close() error {
	b.open = false
	return nil
}

func (b *StdinSource) Read(p []byte) (n int, err error) {
	if b.open {
		return os.Stdin.Read(p)
	} else {
		return 0, io.EOF
	}
}

type FileSource struct {
	name string
	file *os.File
}

func NewFileSource(s string) *FileSource {
	return &FileSource{name: s}
}

func (s *FileSource) Open() error {
	f, err := os.OpenFile(s.name, os.O_RDONLY, 0)
	if err != nil {
		return err
	}
	s.file = f
	return nil
}

func (s *FileSource) Close() error {
	if s.file != nil {
		err := s.file.Close()
		s.file = nil
		return err
	} else {
		return io.EOF
	}
}

func (s *FileSource) Read(p []byte) (n int, err error) {
	if s.file == nil {
		return 0, io.EOF
	} else {
		return s.file.Read(p)
	}
}

type StdoutSink struct {
	open bool
}

func (s *StdoutSink) Open() error {
	s.open = true
	return nil
}

func (s *StdoutSink) Close() error {
	if !s.open {
		return io.EOF
	}
	s.open = false
	return nil
}

func (s *StdoutSink) Write(b []byte) (n int, err error) {
	return os.Stdout.Write(b)
}

func (s *StdoutSink) HitError(e error) error { return nil }

type FileSink struct {
	name string
	file *os.File
	bufw *bufio.Writer
}

func NewFileSink(s string) *FileSink {
	return &FileSink{name: s}
}

func (s *FileSink) Open() error {
	f, err := os.OpenFile(s.name, os.O_WRONLY|os.O_CREATE, UMASKABLE_PERM_FILE)
	if err != nil {
		return fmt.Errorf("Failed to open %s for writing: %s",
			s.name, err.Error())
	}
	s.file = f
	s.bufw = bufio.NewWriter(f)
	return nil
}

func (s *FileSink) Write(b []byte) (n int, err error) {
	if s.bufw == nil {
		return 0, io.EOF
	} else {
		return s.bufw.Write(b)
	}
}

func (s *FileSink) Close() error {
	s.bufw.Flush()
	e := s.file.Close()
{'fraction_non_alphanumeric': 0.14376668720476485, 'fraction_numerical': 0.001643047853768741, 'mean_word_length': 3.3873873873873874, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '578170', 'n_tokens_mistral': 2041, 'n_tokens_neox': 2024, 'n_words': 581}