<commit_before>package manta

import (
	"bufio"
	"encoding/binary"
	"io"
	"os"
	"runtime"
	"strings"

	"code.google.com/p/snappy-go/snappy"

	"github.com/davecgh/go-spew/spew"
	"github.com/dotabuff/manta/dota"
)

func NewParserFromFile(path string) *Parser {
	fd, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	parser := &Parser{
		sendTableId:  -1,
		stream:       bufio.NewReader(fd),
		classInfo:    map[int]string{},
		stringTables: NewStringTables(),
		Callbacks:    &Callbacks{},
	}

	cb := parser.Callbacks

	cb.OnCNETMsg_SpawnGroup_Load = func(m *dota.CNETMsg_SpawnGroup_Load) error { parser.onSpawnGroupLoad(m); return nil }
	cb.OnCSVCMsg_UpdateStringTable = func(m *dota.CSVCMsg_UpdateStringTable) error { parser.stringTables.onUpdateStringTable(m); return nil }
	cb.OnSignonPacket = func(m *dota.CDemoPacket) error { parser.onCDemoPacket(m); return nil }
	cb.OnCDemoClassInfo = func(m *dota.CDemoClassInfo) error { parser.onCDemoClassInfo(m); return nil }
	cb.OnCDemoStop = func(m *dota.CDemoStop) error { parser.Stop(); return nil }
	cb.OnCDemoPacket = func(m *dota.CDemoPacket) error { parser.onCDemoPacket(m); return nil }
	cb.OnCDemoSpawnGroups = func(m *dota.CDemoSpawnGroups) error { parser.onCDemoSpawnGroups(m); return nil }
	cb.OnCDemoStringTables = func(m *dota.CDemoStringTables) error { parser.stringTables.onCDemoStringTables(m); return nil }
	cb.OnCDemoUserCmd = func(m *dota.CDemoUserCmd) error { parser.onCDemoUserCmd(m); return nil }

	if err = parser.readHeader(); err != nil {
		panic(err)
	}

	return parser
}

type DemoHeader struct {
	Magic         [8]byte
	SummaryOffset int32
}

type Parser struct {
	stream       *bufio.Reader
	Header       DemoHeader
	isStopping   bool
	sendTableId  int64
	classInfo    map[int]string
	stringTables *StringTables
	Callbacks    *Callbacks
}

func (p *Parser) Start() {
	for !p.isStopping {
		if msg, err := p.read(); err != nil {
			panic(err)
		} else {
			if err = p.CallByDemoType(int32(msg.Type), msg.data); err != nil {
				panic(err)
			}
		}
	}
}

func (p *Parser) Stop() {
	p.isStopping = true
}

type Message struct {
	Compressed bool
	Tick       uint64
	Type       dota.EDemoCommands
	data       []byte
	Size       uint64
}

var ProtobufDemoSource2Magic = [8]byte{'P', 'B', 'D', 'E', 'M', 'S', '2', '\000'}

func (p *Parser) readHeader() error {
	header := DemoHeader{}
	err := binary.Read(p.stream, binary.LittleEndian, &header)
	if err != nil {
		return err
	}

	if header.Magic != ProtobufDemoSource2Magic {
		panic(spew.Sprintf("Expected %s but got %s", ProtobufDemoSource2Magic, header.Magic))
	}
	p.Header = header

	return nil
}

func (p *Parser) read() (Message, error) {
	msg := Message{}

	binType, err := binary.ReadUvarint(p.stream)
	if err != nil {
		return msg, err
	}

	binTick, err := binary.ReadUvarint(p.stream)
	if err != nil {
		return msg, err
	}
	msg.Tick = binTick
<commit_msg>Add some debugging tools, read a magical 8 bytes after the header - probably additional field(s) in the header..<commit_after>package manta

import (
	"bufio"
	"encoding/binary"
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"

	"github.com/davecgh/go-spew/spew"
	"github.com/dotabuff/manta/dota"
	"github.com/golang/snappy/snappy"
)

func NewParserFromFile(path string) *Parser {
	fd, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	// TODO: look into why we're unable to use bufio.ByteReader with a buffer size > 0.
	// Probably has to do with the different ways we're reading and byte alignment.
	parser := &Parser{
		sendTableId:  -1,
		stream:       bufio.NewReaderSize(fd, 0),
		classInfo:    map[int]string{},
		stringTables: NewStringTables(),
		Callbacks:    &Callbacks{},
	}

	cb := parser.Callbacks

	cb.OnCNETMsg_SpawnGroup_Load = func(m *dota.CNETMsg_SpawnGroup_Load) error { parser.onSpawnGroupLoad(m); return nil }
	cb.OnCSVCMsg_UpdateStringTable = func(m *dota.CSVCMsg_UpdateStringTable) error {
		parser.stringTables.onUpdateStringTable(m)
		return nil
	}
	cb.OnSignonPacket = func(m *dota.CDemoPacket) error { parser.onCDemoPacket(m); return nil }
	cb.OnCDemoClassInfo = func(m *dota.CDemoClassInfo) error { parser.onCDemoClassInfo(m); return nil }
	cb.OnCDemoStop = func(m *dota.CDemoStop) error { parser.Stop(); return nil }
	cb.OnCDemoPacket = func(m *dota.CDemoPacket) error { parser.onCDemoPacket(m); return nil }
	cb.OnCDemoSpawnGroups = func(m *dota.CDemoSpawnGroups) error { parser.onCDemoSpawnGroups(m); return nil }
	cb.OnCDemoStringTables = func(m *dota.CDemoStringTables) error { parser.stringTables.onCDemoStringTables(m); return nil }
	cb.OnCDemoUserCmd = func(m *dota.CDemoUserCmd) error { parser.onCDemoUserCmd(m); return nil }

	if err = parser.readHeader(); err != nil {
		panic(err)
	}

	return parser
}

type DemoHeader struct {
	Magic         [8]byte
	SummaryOffset int32
}

type Parser struct {
	stream       *bufio.Reader
	Header       DemoHeader
	isStopping   bool
	sendTableId  int64
	classInfo    map[int]string
	stringTables *StringTables
	Callbacks    *Callbacks
}

func (p *Parser) Start() {
	for !p.isStopping {
		if msg, err := p.read(); err != nil {
			panic(err)
		} else {
			if err = p.CallByDemoType(int32(msg.Type), msg.data); err != nil {
				// Dump header and data, then panic with the error.
				msg.Dump(-1)
				panic(err)
			}

			// Dump just the header of a successfully parsed message.
			msg.Dump(0)
		}
	}
}

func (p *Parser) Stop() {
	p.isStopping = true
}

type Message struct {
	Compressed bool
	Tick       uint64
	Type       dota.EDemoCommands
	data       []byte
	Size       uint64
}

// Dumps the header of the message, optionally including a preview of the data.
// Preview size of -1 dumps the entire thing, 0 dumps nothing, otherwise length.
func (m *Message) Dump(preview_size int) {
	if preview_size == -1 {
		preview_size = len(m.data)
	}

	if preview_size > len(m.data) {
		preview_size = 0
	}

	fmt.Printf("{tick: %d, type: %v, size: %d, compressed: %v, preview_size: %d}\n", m.Tick, m.Type, m.Size, m.Compressed, preview_size)

	if preview_size > 0 {
		spew.Dump(m.data[:preview_size])
		fmt.Println("-")
	}

}

var ProtobufDemoSource2Magic = [8]byte{'P', 'B', 'D', 'E', 'M', 'S', '2', '\000'}

func (p *Parser) readHeader() error {
	header := DemoHeader{}
	err := binary.Read(p.stream, binary.LittleEndian, &header)
	if err != nil {
		return err
	}

	if header.Magic != ProtobufDemoSource2Magic {
		return fmt.Errorf("expected magic %s, got %s", ProtobufDemoSource2Magic, header.Magic)
	}

	p.Header = header
	spew.Dump(p.Header)

	// TODO: what is this?
	tmp := make([]byte, 8)
	p.stream.Read(tmp)
	spew.Dump(tmp)

	return nil
}

func (p *Parser) read() (Message, error) {
	msg := Message{}

	binType, err := binary.ReadUvarint(p.stream)
	if err != nil {
		return msg, err
	}

	binTick, err := binary.ReadUvarint(p.stream)
	if err != nil {
		return msg, err
	}
	msg.Tick = binTick

{'fraction_non_alphanumeric': 0.13251605370694688, 'fraction_numerical': 0.0074430823117338, 'mean_word_length': 3.7956613016095173, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7403604', 'n_tokens_mistral': 2733, 'n_tokens_neox': 2565, 'n_words': 618}