<commit_before>package octorrent

import (
	"bufio"
	"io"
	"strconv"
)

type decoder struct {
	bufio.Reader
}

func (decoder *decoder) readInt() (interface{}, error) {
	return decoder.readIntUntil('e')
}

func (decoder *decoder) readIntUntil(b byte) (interface{}, error) {
	res, err := decoder.ReadSlice(b)
	if err != nil {
		return -1, err
	}

	// Read until before the 'e'nd
	str := string(res[:len(res)-1])

	if value, err := strconv.ParseInt(str, 10, 64); err == nil {
		return value, nil
	} else if value, err := strconv.ParseUint(str, 10, 64); err == nil {
		return value, nil
	}

	return -1, err
}

func (decoder *decoder) readString() (string, error) {
	// Get length of string
	l, err := decoder.readIntUntil(':')
	if err != nil {
		return "", err
	}

	// Read into buffer
	strLen := l.(int64)
	buf := make([]byte, strLen)
	pos := int64(0)
	for pos < strLen {
		n, err := decoder.Read(buf[pos:])
		if err != nil {
			return "", err
		}

		pos += int64(n)
	}

	return string(buf), nil
}

func (decoder *decoder) readValue() (interface{}, error) {
	b, err := decoder.ReadByte()
	if err != nil {
		return nil, err
	}

	var item interface{}
	switch b {
	case 'i':
		item, err = decoder.readInt()
	case 'l':
		item, err = decoder.readList()
	case 'd':
		item, err = decoder.readDictionary()
	default:
		// Unread last byte, because it belongs to the lenght
		// of the string
		if err := decoder.UnreadByte(); err != nil {
			return nil, err
		}
		item, err = decoder.readString()
	}

	if err != nil {
		return nil, err
	}

	return item, nil
}

func (decoder *decoder) readList() (interface{}, error) {
	var list []interface{}

	for {
		b, err := decoder.ReadByte()
		// Check if
		// - an error occured, if yes return it
		// - the read char is an 'e', if yes, return the list
		// - the unread function was successful, if not return err
		// We need to do UnreadByte to make readValue() work
		if err != nil {
			return nil, err
		} else if b == 'e' {
			return list, nil
		} else if err := decoder.UnreadByte(); err != nil {
			// Unread last byte since it's not an 'e'
			return nil, err
		}

		item, err := decoder.readValue()
		if err != nil {
			return nil, err
		}

		list = append(list, item)
	}

	return list, nil
}

func (decoder *decoder) readDictionary() (map[string]interface{}, error) {
	var dict = map[string]interface{}{}

	//for {
	//key, err := decoder.readString()
	//if err != nil {
	//return nil, err
	//}
	//}

	return dict, nil
}

func Decode(reader io.Reader) (map[string]interface{}, error) {
	return map[string]interface{}{}, nil
}<commit_msg>Add isEnd() for end detection<commit_after>package octorrent

import (
	"bufio"
	"io"
	"strconv"
)

type decoder struct {
	bufio.Reader
}

func (decoder *decoder) readInt() (interface{}, error) {
	return decoder.readIntUntil('e')
}

func (decoder *decoder) readIntUntil(b byte) (interface{}, error) {
	res, err := decoder.ReadSlice(b)
	if err != nil {
		return -1, err
	}

	// Read until before the 'e'nd
	str := string(res[:len(res)-1])

	if value, err := strconv.ParseInt(str, 10, 64); err == nil {
		return value, nil
	} else if value, err := strconv.ParseUint(str, 10, 64); err == nil {
		return value, nil
	}

	return -1, err
}

func (decoder *decoder) readString() (string, error) {
	// Get length of string
	l, err := decoder.readIntUntil(':')
	if err != nil {
		return "", err
	}

	// Read into buffer
	strLen := l.(int64)
	buf := make([]byte, strLen)
	pos := int64(0)
	for pos < strLen {
		n, err := decoder.Read(buf[pos:])
		if err != nil {
			return "", err
		}

		pos += int64(n)
	}

	return string(buf), nil
}

func (decoder *decoder) readValue() (interface{}, error) {
	b, err := decoder.ReadByte()
	if err != nil {
		return nil, err
	}

	var item interface{}
	switch b {
	case 'i':
		item, err = decoder.readInt()
	case 'l':
		item, err = decoder.readList()
	case 'd':
		item, err = decoder.readDictionary()
	default:
		// Unread last byte, because it belongs to the lenght
		// of the string
		if err := decoder.UnreadByte(); err != nil {
			return nil, err
		}
		item, err = decoder.readString()
	}

	if err != nil {
		return nil, err
	}

	return item, nil
}

// Returns if the next byte is an 'e'
func (decoder *decoder) isEnd() (bool, error) {
	b, err := decoder.ReadByte()
	if err != nil {
		return false, err
	} else if b == 'e' {
		return true, nil
	} else if err := decoder.UnreadByte(); err != nil {
		// Unread last byte since it's not an 'e'
		return false, err
	}

	return false, nil
}

func (decoder *decoder) readList() (interface{}, error) {
	var list []interface{}

	for {
		end, err := decoder.isEnd()
		if err != nil {
			return nil, err
		} else if end {
			return list, nil
		}

		item, err := decoder.readValue()
		if err != nil {
			return nil, err
		}

		list = append(list, item)
	}

	return list, nil
}

func (decoder *decoder) readDictionary() (map[string]interface{}, error) {
	var dict = map[string]interface{}{}

	for {
		//key, err := decoder.readString()
		//if err != nil {
		//return nil, err
		//}

	}

	return dict, nil
}

func Decode(reader io.Reader) (map[string]interface{}, error) {
	return map[string]interface{}{}, nil
}
{'fraction_non_alphanumeric': 0.15844660194174756, 'fraction_numerical': 0.006990291262135922, 'mean_word_length': 3.1440064360418343, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6660872', 'n_tokens_mistral': 2038, 'n_tokens_neox': 1831, 'n_words': 608}