<commit_before>// Copyright 2017 The go-sqlite Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sqlite3

import (
	"fmt"
	"io"
	"strings"
)

type btheader struct {
	raw struct {
		Kind PageKind // b-tree page kind

		FreeBlockOffset int16 // byte offset into the page of the first free block
		NCells          int16 // number of cells on this page
		CellsOffset     int16 // offset into first byte of the cell content area
		NFreeBytes      int8  // number of fragmented free bytes within cell area content
	}
}

func (bt btheader) Kind() PageKind {
	return bt.raw.Kind
}

func (bt *btheader) NumCell() int {
	return int(bt.raw.NCells)
}

func (bt *btheader) FreeBlockAddr() int {
	return int(bt.raw.FreeBlockOffset)
}

func (bt *btheader) CellsAddr() int {
	return int(bt.raw.CellsOffset)
}

type btreeTable struct {
	btheader
	db      *DbFile
	pointer int32   // right most pointer (only valid for interior pages
	page    page    // page backing this b-tree leaf
	addrs   []int16 // cell addresses
}

func newBtreeTable(page page, db *DbFile) (*btreeTable, error) {
	var hdr btheader
	if page.ID() == 1 {
		// drop first 100-bytes (global file header)
		_, err := page.Seek(100, 0)
		if err != nil {
			return nil, err
		}
	}

	err := page.Decode(&hdr.raw)
	if err != nil {
		return nil, err
	}

	btree := &btreeTable{
		btheader: hdr,
		db:       db,
		page:     page,
	}

	if btree.Kind() == BTreeInteriorTableKind {
		err = btree.page.Decode(&btree.pointer)
		if err != nil {
			return nil, err
		}
	}

	err = btree.init()
	if err != nil {
		return nil, err
	}

	return btree, err
}

func (btree *btreeTable) ID() int {
	return btree.page.ID()
}

func (btree *btreeTable) Size() int {
	return len(btree.page.buf)
}

func (btree *btreeTable) init() error {
	var err error
	if btree.addrs != nil {
		return nil
	}

	cells := make([]int16, btree.NumCell())
	for icell, addr := range cells {
		// fmt.Printf("   cell= %d/%d... (%d)\n", icell+1, len(cells), btree.page.Pos())
		err = btree.page.Decode(&addr)
		if err != nil {
			return err
		}
		// fmt.Printf("   cell= %d/%d... (%d) => %d\n", icell+1, len(cells), btree.page.Pos(), addr)
		cells[icell] = addr
	}

	btree.addrs = cells
	return err
}

func (btree *btreeTable) decodeRecord(payload []byte) (Record, error) {
	var rec Record

	// decode record
	recbuf := payload[:]
	rhdrsz, n := varint(recbuf)
	if n <= 0 {
		return rec, fmt.Errorf("sqlite3: error decoding record header (n=%d)", n)
	}
	recbuf = recbuf[n:]

	rec.Header.Len = int(rhdrsz) - n
	for ii := 0; ii < rec.Header.Len; {
		v, n := varint(recbuf)
		// fmt.Printf("ii=%d nn=%d len=%d\n", ii, n, rec.Header.Len)
		if n <= 0 {
			return rec, fmt.Errorf("sqlite3: error decoding record header type (n=%d)", n)
		}
		recbuf = recbuf[n:]
		ii += int(n)
		rec.Header.Types = append(rec.Header.Types, SerialType(v))
	}
	rec.Body = recbuf[:]
	//copy(rec.Body, recbuf)

	// fmt.Printf(">>> record: %#v (body=%d)\n", rec.Header, len(rec.Body))
	for _, st := range rec.Header.Types {
		var v interface{}
		switch st {
		case StInt8:
			var vv int8
			n, err := unmarshal(recbuf, &vv)
			if err != nil {
				panic(err)
			}
			recbuf = recbuf[int(n):]
			v = vv

		case StInt16:
			var vv int16
			n, err := unmarshal(recbuf, &vv)
			if err != nil {
				panic(err)
			}
			recbuf = recbuf[int(n):]
			v = vv

		case StInt32:
			var vv int32
			n, err := unmarshal(recbuf, &vv)
			if err != nil {
				panic(err)
			}
			recbuf = recbuf[int(n):]
			v = vv

		case StInt64:
			var vv int64
			n, err := unmarshal(recbuf, &vv)
			if err != nil {
				panic(err)
			}
			recbuf = recbuf[int(n):]
			v = vv

		case StFloat:
			var vv float64
			n, err := unmarshal(recbuf, &vv)
			if err != nil {
				panic(err)
			}
			recbuf = recbuf[int(n):]
			v = vv

		case StC0:
			v = 0

		case StC1:
			v = 1

		case StInt24, StInt48:
			panic("not implemented")

		default:
			if st.IsBlob() {
				vv := make([]byte, st.NBytes())
				n := copy(vv, recbuf)
				recbuf = recbuf[int(n):]
				v = vv
			}
			if st.IsText() {<commit_msg>btree: unmarshal out 3 (as int32) and 6 (as int64) byte ints<commit_after>// Copyright 2017 The go-sqlite Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sqlite3

import (
	"fmt"
	"io"
	"strings"
)

type btheader struct {
	raw struct {
		Kind PageKind // b-tree page kind

		FreeBlockOffset int16 // byte offset into the page of the first free block
		NCells          int16 // number of cells on this page
		CellsOffset     int16 // offset into first byte of the cell content area
		NFreeBytes      int8  // number of fragmented free bytes within cell area content
	}
}

func (bt btheader) Kind() PageKind {
	return bt.raw.Kind
}

func (bt *btheader) NumCell() int {
	return int(bt.raw.NCells)
}

func (bt *btheader) FreeBlockAddr() int {
	return int(bt.raw.FreeBlockOffset)
}

func (bt *btheader) CellsAddr() int {
	return int(bt.raw.CellsOffset)
}

type btreeTable struct {
	btheader
	db      *DbFile
	pointer int32   // right most pointer (only valid for interior pages
	page    page    // page backing this b-tree leaf
	addrs   []int16 // cell addresses
}

func newBtreeTable(page page, db *DbFile) (*btreeTable, error) {
	var hdr btheader
	if page.ID() == 1 {
		// drop first 100-bytes (global file header)
		_, err := page.Seek(100, 0)
		if err != nil {
			return nil, err
		}
	}

	err := page.Decode(&hdr.raw)
	if err != nil {
		return nil, err
	}

	btree := &btreeTable{
		btheader: hdr,
		db:       db,
		page:     page,
	}

	if btree.Kind() == BTreeInteriorTableKind {
		err = btree.page.Decode(&btree.pointer)
		if err != nil {
			return nil, err
		}
	}

	err = btree.init()
	if err != nil {
		return nil, err
	}

	return btree, err
}

func (btree *btreeTable) ID() int {
	return btree.page.ID()
}

func (btree *btreeTable) Size() int {
	return len(btree.page.buf)
}

func (btree *btreeTable) init() error {
	var err error
	if btree.addrs != nil {
		return nil
	}

	cells := make([]int16, btree.NumCell())
	for icell, addr := range cells {
		// fmt.Printf("   cell= %d/%d... (%d)\n", icell+1, len(cells), btree.page.Pos())
		err = btree.page.Decode(&addr)
		if err != nil {
			return err
		}
		// fmt.Printf("   cell= %d/%d... (%d) => %d\n", icell+1, len(cells), btree.page.Pos(), addr)
		cells[icell] = addr
	}

	btree.addrs = cells
	return err
}

func (btree *btreeTable) decodeRecord(payload []byte) (Record, error) {
	var rec Record

	// decode record
	recbuf := payload[:]
	rhdrsz, n := varint(recbuf)
	if n <= 0 {
		return rec, fmt.Errorf("sqlite3: error decoding record header (n=%d)", n)
	}
	recbuf = recbuf[n:]

	rec.Header.Len = int(rhdrsz) - n
	for ii := 0; ii < rec.Header.Len; {
		v, n := varint(recbuf)
		// fmt.Printf("ii=%d nn=%d len=%d\n", ii, n, rec.Header.Len)
		if n <= 0 {
			return rec, fmt.Errorf("sqlite3: error decoding record header type (n=%d)", n)
		}
		recbuf = recbuf[n:]
		ii += int(n)
		rec.Header.Types = append(rec.Header.Types, SerialType(v))
	}
	rec.Body = recbuf[:]
	//copy(rec.Body, recbuf)

	// fmt.Printf(">>> record: %#v (body=%d)\n", rec.Header, len(rec.Body))
	for _, st := range rec.Header.Types {
		var v interface{}
		switch st {
		case StInt8:
			var vv int8
			n, err := unmarshal(recbuf, &vv)
			if err != nil {
				panic(err)
			}
			recbuf = recbuf[int(n):]
			v = vv

		case StInt16:
			var vv int16
			n, err := unmarshal(recbuf, &vv)
			if err != nil {
				panic(err)
			}
			recbuf = recbuf[int(n):]
			v = vv

		case StInt24:
			bs := make([]byte, 3)
			n64, n := varint(bs)
			var vv int32 = int32(n64)
			recbuf = recbuf[int(n):]
			v = vv

		case StInt32:
			var vv int32
			n, err := unmarshal(recbuf, &vv)
			if err != nil {
				panic(err)
			}
			recbuf = recbuf[int(n):]
			v = vv

		case StInt48:
			bs := make([]byte, 6)
			n64, n := varint(bs)
			var vv int64 = int64(n64)
			recbuf = recbuf[int(n):]
			v = vv

		case StInt64:
			var vv int64
			n, err := unmarshal(recbuf, &vv)
			if err != nil {
				panic(err)
			}
			recbuf = recbuf[int(n):]
			v = vv

		case StFloat:
			var vv float64
			n, err := unmarshal(recbuf, &vv)
			if err != nil {
				panic(err)
			}
			recbuf = recbuf[int(n):]
			v = vv

		case StC0:
			v = 0

		case StC1:
			v = 1

		default:
			if st.IsBlob() {
				vv := make([]byte, st.NBytes())
				n := copy(vv, recbuf)
				recbuf = recbuf[int(n):]
				v = vv
			}
			if st.IsText() {
{'fraction_non_alphanumeric': 0.14830059978831, 'fraction_numerical': 0.016229566035516875, 'mean_word_length': 2.7347386912604303, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6655352', 'n_tokens_mistral': 3785, 'n_tokens_neox': 3323, 'n_words': 980}