<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements buffered I/O.  It wraps an io.Reader or io.Writer
// object, creating another object (Reader or Writer) that also implements
// the interface but provides buffering and some help for textual I/O.
package bufio

import (
	"bytes"
	"io"
	"os"
	"strconv"
	"utf8"
)


const (
	defaultBufSize = 4096
)

// Errors introduced by this package.
type Error struct {
	os.ErrorString
}

var (
	ErrInvalidUnreadByte os.Error = &Error{"bufio: invalid use of UnreadByte"}
	ErrInvalidUnreadRune os.Error = &Error{"bufio: invalid use of UnreadRune"}
	ErrBufferFull        os.Error = &Error{"bufio: buffer full"}
	ErrNegativeCount     os.Error = &Error{"bufio: negative count"}
	errInternal          os.Error = &Error{"bufio: internal error"}
)

// BufSizeError is the error representing an invalid buffer size.
type BufSizeError int

func (b BufSizeError) String() string {
	return "bufio: bad buffer size " + strconv.Itoa(int(b))
}


// Buffered input.

// Reader implements buffering for an io.Reader object.
type Reader struct {
	buf          []byte
	rd           io.Reader
	r, w         int
	err          os.Error
	lastByte     int
	lastRuneSize int
}

// NewReaderSize creates a new Reader whose buffer has the specified size,
// which must be greater than zero.  If the argument io.Reader is already a
// Reader with large enough size, it returns the underlying Reader.
// It returns the Reader and any error.
func NewReaderSize(rd io.Reader, size int) (*Reader, os.Error) {
	if size <= 0 {
		return nil, BufSizeError(size)
	}
	// Is it already a Reader?
	b, ok := rd.(*Reader)
	if ok && len(b.buf) >= size {
		return b, nil
	}
	b = new(Reader)
	b.buf = make([]byte, size)
	b.rd = rd
	b.lastByte = -1
	b.lastRuneSize = -1
	return b, nil
}

// NewReader returns a new Reader whose buffer has the default size.
func NewReader(rd io.Reader) *Reader {
	b, err := NewReaderSize(rd, defaultBufSize)
	if err != nil {
		// cannot happen - defaultBufSize is a valid size
		panic(err)
	}
	return b
}

// fill reads a new chunk into the buffer.
func (b *Reader) fill() {
	// Slide existing data to beginning.
	if b.r > 0 {
		copy(b.buf, b.buf[b.r:b.w])
		b.w -= b.r
		b.r = 0
	}

	// Read new data.
	n, e := b.rd.Read(b.buf[b.w:])
	b.w += n
	if e != nil {
		b.err = e
	}
}

// Peek returns the next n bytes without advancing the reader. The bytes stop
// being valid at the next read call. If Peek returns fewer than n bytes, it
// also returns an error explaining why the read is short. The error is
// ErrBufferFull if n is larger than b's buffer size.
func (b *Reader) Peek(n int) ([]byte, os.Error) {
	if n < 0 {
		return nil, ErrNegativeCount
	}
	if n > len(b.buf) {
		return nil, ErrBufferFull
	}
	for b.w-b.r < n && b.err == nil {
		b.fill()
	}
	m := b.w - b.r
	if m > n {
		m = n
	}
	err := b.err
	if m < n && err == nil {
		err = ErrBufferFull
	}
	return b.buf[b.r : b.r+m], err
}

// Read reads data into p.
// It returns the number of bytes read into p.
// If nn < len(p), also returns an error explaining
// why the read is short.  At EOF, the count will be
// zero and err will be os.EOF.
func (b *Reader) Read(p []byte) (nn int, err os.Error) {
	nn = 0
	for len(p) > 0 {
		n := len(p)
		if b.w == b.r {
			if b.err != nil {
				return nn, b.err
			}
			if len(p) >= len(b.buf) {
				// Large read, empty buffer.
				// Read directly into p to avoid copy.
				n, b.err = b.rd.Read(p)
				if n > 0 {
					b.lastByte = int(p[n-1])
					b.lastRuneSize = -1
				}
				p = p[n:]
				nn += n
				continue
			}
			b.fill()
			continue
		}
		if n > b.w-b.r {
			n = b.w - b.r
		}
		copy(p[0:n], b.buf[b.r:])
		p = p[n:]
		b.r += n
		b.lastByte = int(b.buf[b.r-1])
		b.lastRuneSize = -1
		nn += n
	}
	return nn, nil
}

// ReadByte reads and returns a single byte.
// If no byte is available, returns an error.
func (b *Reader) ReadByte() (c byte, err os.Error) {
	b.lastRuneSize = -1
	for b.w == b.r {
		if b.err != nil {
			return 0, b.err
		}
		b.fill()
	}
	c = b.buf[b.r]
	b.r++
	b.lastByte = int(c)
	return c, nil
}

// UnreadByte unreads the last byte.  Only the most recently read byte can be unread.
func (b *Reader) UnreadByte() os.Error {
	b.lastRuneSize = -1
	if b.r == b.w && b.lastByte >= 0 {
		b.w = 1<commit_msg>bufio: make Reader.Read implement io.Reader semantics<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements buffered I/O.  It wraps an io.Reader or io.Writer
// object, creating another object (Reader or Writer) that also implements
// the interface but provides buffering and some help for textual I/O.
package bufio

import (
	"bytes"
	"io"
	"os"
	"strconv"
	"utf8"
)


const (
	defaultBufSize = 4096
)

// Errors introduced by this package.
type Error struct {
	os.ErrorString
}

var (
	ErrInvalidUnreadByte os.Error = &Error{"bufio: invalid use of UnreadByte"}
	ErrInvalidUnreadRune os.Error = &Error{"bufio: invalid use of UnreadRune"}
	ErrBufferFull        os.Error = &Error{"bufio: buffer full"}
	ErrNegativeCount     os.Error = &Error{"bufio: negative count"}
	errInternal          os.Error = &Error{"bufio: internal error"}
)

// BufSizeError is the error representing an invalid buffer size.
type BufSizeError int

func (b BufSizeError) String() string {
	return "bufio: bad buffer size " + strconv.Itoa(int(b))
}


// Buffered input.

// Reader implements buffering for an io.Reader object.
type Reader struct {
	buf          []byte
	rd           io.Reader
	r, w         int
	err          os.Error
	lastByte     int
	lastRuneSize int
}

// NewReaderSize creates a new Reader whose buffer has the specified size,
// which must be greater than zero.  If the argument io.Reader is already a
// Reader with large enough size, it returns the underlying Reader.
// It returns the Reader and any error.
func NewReaderSize(rd io.Reader, size int) (*Reader, os.Error) {
	if size <= 0 {
		return nil, BufSizeError(size)
	}
	// Is it already a Reader?
	b, ok := rd.(*Reader)
	if ok && len(b.buf) >= size {
		return b, nil
	}
	b = new(Reader)
	b.buf = make([]byte, size)
	b.rd = rd
	b.lastByte = -1
	b.lastRuneSize = -1
	return b, nil
}

// NewReader returns a new Reader whose buffer has the default size.
func NewReader(rd io.Reader) *Reader {
	b, err := NewReaderSize(rd, defaultBufSize)
	if err != nil {
		// cannot happen - defaultBufSize is a valid size
		panic(err)
	}
	return b
}

// fill reads a new chunk into the buffer.
func (b *Reader) fill() {
	// Slide existing data to beginning.
	if b.r > 0 {
		copy(b.buf, b.buf[b.r:b.w])
		b.w -= b.r
		b.r = 0
	}

	// Read new data.
	n, e := b.rd.Read(b.buf[b.w:])
	b.w += n
	if e != nil {
		b.err = e
	}
}

// Peek returns the next n bytes without advancing the reader. The bytes stop
// being valid at the next read call. If Peek returns fewer than n bytes, it
// also returns an error explaining why the read is short. The error is
// ErrBufferFull if n is larger than b's buffer size.
func (b *Reader) Peek(n int) ([]byte, os.Error) {
	if n < 0 {
		return nil, ErrNegativeCount
	}
	if n > len(b.buf) {
		return nil, ErrBufferFull
	}
	for b.w-b.r < n && b.err == nil {
		b.fill()
	}
	m := b.w - b.r
	if m > n {
		m = n
	}
	err := b.err
	if m < n && err == nil {
		err = ErrBufferFull
	}
	return b.buf[b.r : b.r+m], err
}

// Read reads data into p.
// It returns the number of bytes read into p.
// It calls Read at most once on the underlying Reader,
// hence n may be less than len(p).
// At EOF, the count will be zero and err will be os.EOF.
func (b *Reader) Read(p []byte) (n int, err os.Error) {
	n = len(p)
	if n == 0 {
		return 0, b.err
	}
	if b.w == b.r {
		if b.err != nil {
			return 0, b.err
		}
		if len(p) >= len(b.buf) {
			// Large read, empty buffer.
			// Read directly into p to avoid copy.
			n, b.err = b.rd.Read(p)
			if n > 0 {
				b.lastByte = int(p[n-1])
				b.lastRuneSize = -1
			}
			p = p[n:]
			return n, b.err
		}
		b.fill()
		if b.w == b.r {
			return 0, b.err
		}
	}

	if n > b.w-b.r {
		n = b.w - b.r
	}
	copy(p[0:n], b.buf[b.r:])
	p = p[n:]
	b.r += n
	b.lastByte = int(b.buf[b.r-1])
	b.lastRuneSize = -1
	return n, nil
}

// ReadByte reads and returns a single byte.
// If no byte is available, returns an error.
func (b *Reader) ReadByte() (c byte, err os.Error) {
	b.lastRuneSize = -1
	for b.w == b.r {
		if b.err != nil {
			return 0, b.err
		}
		b.fill()
	}
	c = b.buf[b.r]
	b.r++
	b.lastByte = int(c)
	return c, nil
}

// UnreadByte unreads the last byte.  Only the most recently read byte can be unread.
func (b *Reader) UnreadByte() os.Error {
	b.lastRuneSize = -1
	if b.r == b.w && b.lastByte >= 0 {
		b.w = 1
{'fraction_non_alphanumeric': 0.1305427056301478, 'fraction_numerical': 0.0065440595735078416, 'mean_word_length': 2.5727529222087866, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3477953', 'n_tokens_mistral': 3520, 'n_tokens_neox': 3293, 'n_words': 1204}