<commit_before>package gob

// This file is not normally included in the gob package.  Used only for debugging the package itself.
// Add debug.go to the files listed in the Makefile to add Debug to the gob package.
// Except for reading uints, it is an implementation of a reader that is independent of
// the one implemented by Decoder.

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"strings"
	"sync"
)

var dumpBytes = false // If true, print the remaining bytes in the input buffer at each item.

// Init installs the debugging facility. If this file is not compiled in the
// package, the tests in codec_test.go are no-ops.
func init() {
	debugFunc = Debug
}

var (
	blanks = bytes.Repeat([]byte{' '}, 3*10)
	empty  = []byte(": <empty>\n")
	tabs   = strings.Repeat("\t", 100)
)

// tab indents itself when printed.
type tab int

func (t tab) String() string {
	n := int(t)
	if n > len(tabs) {
		n = len(tabs)
	}
	return tabs[0:n]
}

func (t tab) print() {
	fmt.Fprint(os.Stderr, t)
}

// A peekReader wraps an io.Reader, allowing one to peek ahead to see
// what's coming without stealing the data from the client of the Reader.
type peekReader struct {
	r    io.Reader
	data []byte // read-ahead data
}

// newPeekReader returns a peekReader that wraps r.
func newPeekReader(r io.Reader) *peekReader {
	return &peekReader{r: r}
}

// Read is the usual method. It will first take data that has been read ahead.
func (p *peekReader) Read(b []byte) (n int, err os.Error) {
	if len(p.data) == 0 {
		return p.r.Read(b)
	}
	// Satisfy what's possible from the read-ahead data.
	n = copy(b, p.data)
	// Move data down to beginning of slice, to avoid endless growth
	copy(p.data, p.data[n:])
	p.data = p.data[:len(p.data)-n]
	return
}

// peek returns as many bytes as possible from the unread
// portion of the stream, up to the length of b.
func (p *peekReader) peek(b []byte) (n int, err os.Error) {
	if len(p.data) > 0 {
		n = copy(b, p.data)
		if n == len(b) {
			return
		}
		b = b[n:]
	}
	if len(b) == 0 {
		return
	}
	m, e := io.ReadFull(p.r, b)
	if m > 0 {
		p.data = append(p.data, b[:m]...)
	}
	n += m
	if e == io.ErrUnexpectedEOF {
		// That means m > 0 but we reached EOF. If we got data
		// we won't complain about not being able to peek enough.
		if n > 0 {
			e = nil
		} else {
			e = os.EOF
		}
	}
	return n, e
}

type debugger struct {
	mutex          sync.Mutex
	remain         int  // the number of bytes known to remain in the input
	remainingKnown bool // the value of 'remain' is valid
	r              *peekReader
	wireType       map[typeId]*wireType
	tmp            []byte // scratch space for decoding uints.
}

// dump prints the next nBytes of the input.
// It arranges to print the output aligned from call to
// call, to make it easy to see what has been consumed.
func (deb *debugger) dump(format string, args ...interface{}) {
	if !dumpBytes {
		return
	}
	fmt.Fprintf(os.Stderr, format+" ", args...)
	if !deb.remainingKnown {
		return
	}
	if deb.remain < 0 {
		fmt.Fprintf(os.Stderr, "remaining byte count is negative! %d\n", deb.remain)
		return
	}
	data := make([]byte, deb.remain)
	n, _ := deb.r.peek(data)
	if n == 0 {
		os.Stderr.Write(empty)
		return
	}
	b := new(bytes.Buffer)
	fmt.Fprintf(b, "[%d]{\n", deb.remain)
	// Blanks until first byte
	lineLength := 0
	if n := len(data); n%10 != 0 {
		lineLength = 10 - n%10
		fmt.Fprintf(b, "\t%s", blanks[:lineLength*3])
	}
	// 10 bytes per line
	for len(data) > 0 {
		if lineLength == 0 {
			fmt.Fprint(b, "\t")
		}
		m := 10 - lineLength
		lineLength = 0
		if m > len(data) {
			m = len(data)
		}
		fmt.Fprintf(b, "% x\n", data[:m])
		data = data[m:]
	}
	fmt.Fprint(b, "}\n")
	os.Stderr.Write(b.Bytes())
}

// Debug prints a human-readable representation of the gob data read from r.
func Debug(r io.Reader) {
	err := debug(r)
	if err != nil {
		fmt.Fprintf(os.Stderr, "gob debug: %s\n", err)
	}
}

// debug implements Debug, but catches panics and returns
// them as errors to be printed by Debug.
func debug(r io.Reader) (err os.Error) {
	defer catchError(&err)
	fmt.Fprintln(os.Stderr, "Start of debugging")
	deb := &debugger{
		r:        newPeekReader(r),
		wireType: make(map[typeId]*wireType),
		tmp:      make([]byte, 16),
	}
	if b, ok := r.(*bytes.Buffer); ok {
		deb.remain = b.Len()
		deb.remainingKnown = true
	}
	deb.gobStream()
	return
}

// note that we've consumed some bytes
func (deb *debugger) consumed(n int) {<commit_msg>gob: explain that Debug isn't useful unless it's compiled in.<commit_after>package gob

// This file is not normally included in the gob package.  Used only for debugging the package itself.
// Add debug.go to the files listed in the Makefile to add Debug to the gob package.
// Except for reading uints, it is an implementation of a reader that is independent of
// the one implemented by Decoder.

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"strings"
	"sync"
)

var dumpBytes = false // If true, print the remaining bytes in the input buffer at each item.

// Init installs the debugging facility. If this file is not compiled in the
// package, the tests in codec_test.go are no-ops.
func init() {
	debugFunc = Debug
}

var (
	blanks = bytes.Repeat([]byte{' '}, 3*10)
	empty  = []byte(": <empty>\n")
	tabs   = strings.Repeat("\t", 100)
)

// tab indents itself when printed.
type tab int

func (t tab) String() string {
	n := int(t)
	if n > len(tabs) {
		n = len(tabs)
	}
	return tabs[0:n]
}

func (t tab) print() {
	fmt.Fprint(os.Stderr, t)
}

// A peekReader wraps an io.Reader, allowing one to peek ahead to see
// what's coming without stealing the data from the client of the Reader.
type peekReader struct {
	r    io.Reader
	data []byte // read-ahead data
}

// newPeekReader returns a peekReader that wraps r.
func newPeekReader(r io.Reader) *peekReader {
	return &peekReader{r: r}
}

// Read is the usual method. It will first take data that has been read ahead.
func (p *peekReader) Read(b []byte) (n int, err os.Error) {
	if len(p.data) == 0 {
		return p.r.Read(b)
	}
	// Satisfy what's possible from the read-ahead data.
	n = copy(b, p.data)
	// Move data down to beginning of slice, to avoid endless growth
	copy(p.data, p.data[n:])
	p.data = p.data[:len(p.data)-n]
	return
}

// peek returns as many bytes as possible from the unread
// portion of the stream, up to the length of b.
func (p *peekReader) peek(b []byte) (n int, err os.Error) {
	if len(p.data) > 0 {
		n = copy(b, p.data)
		if n == len(b) {
			return
		}
		b = b[n:]
	}
	if len(b) == 0 {
		return
	}
	m, e := io.ReadFull(p.r, b)
	if m > 0 {
		p.data = append(p.data, b[:m]...)
	}
	n += m
	if e == io.ErrUnexpectedEOF {
		// That means m > 0 but we reached EOF. If we got data
		// we won't complain about not being able to peek enough.
		if n > 0 {
			e = nil
		} else {
			e = os.EOF
		}
	}
	return n, e
}

type debugger struct {
	mutex          sync.Mutex
	remain         int  // the number of bytes known to remain in the input
	remainingKnown bool // the value of 'remain' is valid
	r              *peekReader
	wireType       map[typeId]*wireType
	tmp            []byte // scratch space for decoding uints.
}

// dump prints the next nBytes of the input.
// It arranges to print the output aligned from call to
// call, to make it easy to see what has been consumed.
func (deb *debugger) dump(format string, args ...interface{}) {
	if !dumpBytes {
		return
	}
	fmt.Fprintf(os.Stderr, format+" ", args...)
	if !deb.remainingKnown {
		return
	}
	if deb.remain < 0 {
		fmt.Fprintf(os.Stderr, "remaining byte count is negative! %d\n", deb.remain)
		return
	}
	data := make([]byte, deb.remain)
	n, _ := deb.r.peek(data)
	if n == 0 {
		os.Stderr.Write(empty)
		return
	}
	b := new(bytes.Buffer)
	fmt.Fprintf(b, "[%d]{\n", deb.remain)
	// Blanks until first byte
	lineLength := 0
	if n := len(data); n%10 != 0 {
		lineLength = 10 - n%10
		fmt.Fprintf(b, "\t%s", blanks[:lineLength*3])
	}
	// 10 bytes per line
	for len(data) > 0 {
		if lineLength == 0 {
			fmt.Fprint(b, "\t")
		}
		m := 10 - lineLength
		lineLength = 0
		if m > len(data) {
			m = len(data)
		}
		fmt.Fprintf(b, "% x\n", data[:m])
		data = data[m:]
	}
	fmt.Fprint(b, "}\n")
	os.Stderr.Write(b.Bytes())
}

// Debug prints a human-readable representation of the gob data read from r.
// It is a no-op unless debugging was enabled when the package was built.
func Debug(r io.Reader) {
	err := debug(r)
	if err != nil {
		fmt.Fprintf(os.Stderr, "gob debug: %s\n", err)
	}
}

// debug implements Debug, but catches panics and returns
// them as errors to be printed by Debug.
func debug(r io.Reader) (err os.Error) {
	defer catchError(&err)
	fmt.Fprintln(os.Stderr, "Start of debugging")
	deb := &debugger{
		r:        newPeekReader(r),
		wireType: make(map[typeId]*wireType),
		tmp:      make([]byte, 16),
	}
	if b, ok := r.(*bytes.Buffer); ok {
		deb.remain = b.Len()
		deb.remainingKnown = true
	}
	deb.gobStream()
	return
}

// note that we've consumed some bytes
func (deb *debugger) consumed(n int) {
{'fraction_non_alphanumeric': 0.13724615041285426, 'fraction_numerical': 0.0073644275831287655, 'mean_word_length': 2.8851322063285654, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2841495', 'n_tokens_mistral': 3489, 'n_tokens_neox': 3267, 'n_words': 1168}