<commit_before>package fit

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"math"
	"reflect"
	"sort"
	"time"

	"github.com/tormoder/fit/dyncrc16"
	"github.com/tormoder/fit/internal/types"
)

var (
	le = binary.LittleEndian
	be = binary.BigEndian
)

type decoder struct {
	r     io.Reader
	bytes struct {
		limit int
		n     int
		buf   [4096]byte
		i, j  int
	}

	crc     dyncrc16.Hash16
	tmp     [maxFieldSize]byte
	defmsgs [maxLocalMesgs]*defmsg

	timestamp      uint32
	lastTimeOffset int32

	opts  decodeOptions
	debug bool

	unknownFields   map[unknownField]int
	unknownMessages map[MesgNum]int

	h   Header
	fit *Fit
}

// CheckIntegrity verifies the FIT header and file CRC. Only the header CRC is
// verified if headerOnly is true.
func CheckIntegrity(r io.Reader, headerOnly bool) error {
	var d decoder
	return d.decode(r, headerOnly, false, true)
}

// DecodeHeader returns the FIT file header without decoding the entire FIT
// file.
func DecodeHeader(r io.Reader) (Header, error) {
	var d decoder
	if err := d.decode(r, true, false, false); err != nil {
		return Header{}, err
	}
	return d.h, nil
}

// DecodeHeaderAndFileID returns the FIT file header and FileId message without
// decoding the entire FIT file. The FileId message must be present in all FIT
// files.
func DecodeHeaderAndFileID(r io.Reader) (Header, FileIdMsg, error) {
	var d decoder
	if err := d.decode(r, false, true, false); err != nil {
		return Header{}, FileIdMsg{}, err
	}
	return d.h, d.fit.FileId, nil
}

// Decode reads a FIT file from r and returns it as a *Fit.
// If error is non-nil, all data decoded before the error was
// encountered is also returned.
func Decode(r io.Reader, opts ...DecodeOption) (*Fit, error) {
	var d decoder
	for _, opt := range opts {
		opt(&d.opts)
	}
	err := d.decode(r, false, false, false)
	return d.fit, err
}

func (d *decoder) decode(r io.Reader, headerOnly, fileIDOnly, crcOnly bool) error {
	if d.opts.logger != nil {
		d.debug = true
	}

	d.r = r
	d.crc = dyncrc16.New()

	err := d.decodeHeader()
	if err != nil {
		return fmt.Errorf("error decoding header: %v", err)
	}

	d.fit = new(Fit)
	d.fit.Header = d.h
	d.bytes.limit = int(d.h.DataSize)

	if d.debug {
		d.opts.logger.Println("header decoded:", d.h)
	}

	if headerOnly {
		return nil
	}

	if crcOnly {
		_, err = io.CopyN(d.crc, d.r, int64(d.h.DataSize))
		if err != nil {
			return fmt.Errorf("error parsing data: %v", err)
		}
		return d.checkCRC()
	}

	if d.opts.unknownFields {
		d.unknownFields = make(map[unknownField]int)
		defer d.handleUnknownFields()
	}
	if d.opts.unknownMessages {
		d.unknownMessages = make(map[MesgNum]int)
		defer d.handleUnknownMessages()
	}

	err = d.parseFileIdMsg()
	if err != nil {
		return fmt.Errorf("error parsing file id message: %v", err)
	}
	if fileIDOnly {
		return nil
	}

	err = d.initFileType()
	if err != nil {
		return err
	}

	for d.bytes.n < d.bytes.limit {
		var (
			b   byte
			dm  *defmsg
			msg reflect.Value
		)

		b, err = d.readByte()
		if err != nil {
			return fmt.Errorf("error parsing record header: %v", err)
		}

		switch {

		case (b & compressedHeaderMask) == compressedHeaderMask:
			msg, err = d.parseDataMessage(b, true)
			if err != nil {
				return fmt.Errorf("compressed timestamp message: %v", err)
			}
			if msg.IsValid() {
				d.fit.add(msg)
			}
		case (b & headerTypeMask) == mesgDefinitionMask:
			dm, err = d.parseDefinitionMessage(b)
			if err != nil {
				return fmt.Errorf("parsing definition message: %v", err)
			}
			d.defmsgs[dm.localMsgType] = dm
		case (b & mesgHeaderMask) == mesgHeaderMask:
			msg, err = d.parseDataMessage(b, false)
			if err != nil {
				return fmt.Errorf("parsing data message: %v", err)
			}
			if msg.IsValid() {
				d.fit.add(msg)
			}
		default:
			return fmt.Errorf("unknown record header, got: %#x", b)
		}
	}

	// Check invariant pre-read CRC:
	if !crcOnly && d.bytes.n != d.bytes.limit {
		fatalErr := fmt.Sprintf("internal decoder error: pre-crc check: data size limit is %d, but n is %d", d.bytes.limit, d.bytes.n)
		panic(fatalErr)
	}

	return d.checkCRC()
}

func (d *decoder) checkCRC() error {
	if d.debug {
		d.opts.logger.Printf("expecting crc value: 0x%x", d.crc.Sum16())
	}
	if _, err := io.ReadFull(d.r, d.tmp[:bytesForCRC]); err != nil {
		err = noEOF(err)
		return fmt.Errorf("error parsing file CRC: %v", err)
	}
	d.crc.Write(d.tmp[:bytesForCRC])
	d.fit.CRC = le.Uint16(d.tmp[:bytesForCRC])
	if d.debug {
		d.opts.logger.Printf("read crc value: 0x%x", d.fit.CRC)

	}
	if d.crc.Sum16() != 0x0000 {
		return IntegrityError("file checksum failed")
	}

	return nil
}
<commit_msg>reader: extract main decode loop as separate method<commit_after>package fit

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"math"
	"reflect"
	"sort"
	"time"

	"github.com/tormoder/fit/dyncrc16"
	"github.com/tormoder/fit/internal/types"
)

var (
	le = binary.LittleEndian
	be = binary.BigEndian
)

type decoder struct {
	r     io.Reader
	bytes struct {
		limit int
		n     int
		buf   [4096]byte
		i, j  int
	}

	crc     dyncrc16.Hash16
	tmp     [maxFieldSize]byte
	defmsgs [maxLocalMesgs]*defmsg

	timestamp      uint32
	lastTimeOffset int32

	opts  decodeOptions
	debug bool

	unknownFields   map[unknownField]int
	unknownMessages map[MesgNum]int

	h   Header
	fit *Fit
}

// CheckIntegrity verifies the FIT header and file CRC. Only the header CRC is
// verified if headerOnly is true.
func CheckIntegrity(r io.Reader, headerOnly bool) error {
	var d decoder
	return d.decode(r, headerOnly, false, true)
}

// DecodeHeader returns the FIT file header without decoding the entire FIT
// file.
func DecodeHeader(r io.Reader) (Header, error) {
	var d decoder
	if err := d.decode(r, true, false, false); err != nil {
		return Header{}, err
	}
	return d.h, nil
}

// DecodeHeaderAndFileID returns the FIT file header and FileId message without
// decoding the entire FIT file. The FileId message must be present in all FIT
// files.
func DecodeHeaderAndFileID(r io.Reader) (Header, FileIdMsg, error) {
	var d decoder
	if err := d.decode(r, false, true, false); err != nil {
		return Header{}, FileIdMsg{}, err
	}
	return d.h, d.fit.FileId, nil
}

// Decode reads a FIT file from r and returns it as a *Fit.
// If error is non-nil, all data decoded before the error was
// encountered is also returned.
func Decode(r io.Reader, opts ...DecodeOption) (*Fit, error) {
	var d decoder
	for _, opt := range opts {
		opt(&d.opts)
	}
	err := d.decode(r, false, false, false)
	return d.fit, err
}

func (d *decoder) decode(r io.Reader, headerOnly, fileIDOnly, crcOnly bool) error {
	if d.opts.logger != nil {
		d.debug = true
	}

	d.r = r
	d.crc = dyncrc16.New()

	err := d.decodeHeader()
	if err != nil {
		return fmt.Errorf("error decoding header: %v", err)
	}

	d.fit = new(Fit)
	d.fit.Header = d.h
	d.bytes.limit = int(d.h.DataSize)

	if d.debug {
		d.opts.logger.Println("header decoded:", d.h)
	}

	if headerOnly {
		return nil
	}

	if crcOnly {
		_, err = io.CopyN(d.crc, d.r, int64(d.h.DataSize))
		if err != nil {
			return fmt.Errorf("error parsing data: %v", err)
		}
		return d.checkCRC()
	}

	if d.opts.unknownFields {
		d.unknownFields = make(map[unknownField]int)
		defer d.handleUnknownFields()
	}
	if d.opts.unknownMessages {
		d.unknownMessages = make(map[MesgNum]int)
		defer d.handleUnknownMessages()
	}

	err = d.parseFileIdMsg()
	if err != nil {
		return fmt.Errorf("error parsing file id message: %v", err)
	}
	if fileIDOnly {
		return nil
	}

	err = d.initFileType()
	if err != nil {
		return err
	}

	err = d.decodeFileData()
	if err != nil {
		return err
	}

	// Check invariant pre-read CRC:
	if !crcOnly && d.bytes.n != d.bytes.limit {
		fatalErr := fmt.Sprintf("internal decoder error: pre-crc check: data size limit is %d, but n is %d", d.bytes.limit, d.bytes.n)
		panic(fatalErr)
	}

	return d.checkCRC()
}

func (d *decoder) decodeFileData() error {
	for d.bytes.n < d.bytes.limit {
		var (
			b   byte
			dm  *defmsg
			msg reflect.Value
			err error
		)

		b, err = d.readByte()
		if err != nil {
			return fmt.Errorf("error parsing record header: %v", err)
		}

		switch {

		case (b & compressedHeaderMask) == compressedHeaderMask:
			msg, err = d.parseDataMessage(b, true)
			if err != nil {
				return fmt.Errorf("compressed timestamp message: %v", err)
			}
			if msg.IsValid() {
				d.fit.add(msg)
			}
		case (b & headerTypeMask) == mesgDefinitionMask:
			dm, err = d.parseDefinitionMessage(b)
			if err != nil {
				return fmt.Errorf("parsing definition message: %v", err)
			}
			d.defmsgs[dm.localMsgType] = dm
		case (b & mesgHeaderMask) == mesgHeaderMask:
			msg, err = d.parseDataMessage(b, false)
			if err != nil {
				return fmt.Errorf("parsing data message: %v", err)
			}
			if msg.IsValid() {
				d.fit.add(msg)
			}
		default:
			return fmt.Errorf("unknown record header, got: %#x", b)
		}
	}

	return nil
}

func (d *decoder) checkCRC() error {
	if d.debug {
		d.opts.logger.Printf("expecting crc value: 0x%x", d.crc.Sum16())
	}
	if _, err := io.ReadFull(d.r, d.tmp[:bytesForCRC]); err != nil {
		err = noEOF(err)
		return fmt.Errorf("error parsing file CRC: %v", err)
	}
	d.crc.Write(d.tmp[:bytesForCRC])
	d.fit.CRC = le.Uint16(d.tmp[:bytesForCRC])
	if d.debug {
		d.opts.logger.Printf("read crc value: 0x%x", d.fit.CRC)

	}
	if d.crc.Sum16() != 0x0000 {
		return IntegrityError("file checksum failed")
	}

	return nil
}

{'fraction_non_alphanumeric': 0.13376, 'fraction_numerical': 0.006613333333333333, 'mean_word_length': 3.2044843049327354, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '587416', 'n_tokens_mistral': 3836, 'n_tokens_neox': 3490, 'n_words': 993}