<commit_before>package php_session_decoder

import (
	"bytes"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"log"
)

type PhpDecoder struct {
	DecodeFunc	SerializableDecodeFunc
	RawData		string
	source		*strings.Reader
	data		PhpSessionData
}

func NewPhpDecoder(phpSession string) *PhpDecoder {
	sessionData := make(PhpSessionData)
	d := &PhpDecoder{
		RawData:	phpSession,
		source:		strings.NewReader(phpSession),
		data:		sessionData,
	}
	return d
}

func (decoder *PhpDecoder) Decode() (PhpSessionData, error) {
	var resultErr error
	for {
		if valueName, err := decoder.readUntil(VALUE_NAME_SEPARATOR); err == nil {
			if value, err := decoder.DecodeValue(); err == nil {
				decoder.data[valueName] = value
			} else {
				resultErr = fmt.Errorf("Can not read variable(%v) value:%v", valueName, err)
				break
			}
		} else {
			break
		}
	}
	return decoder.data, resultErr
}

func (decoder *PhpDecoder) DecodeValue() (PhpValue, error) {
	var (
		value PhpValue
		err   error
	)

	if token, _, err := decoder.source.ReadRune(); err == nil {
		decoder.expect(TYPE_VALUE_SEPARATOR)
		switch token {
		case 'N':
			value = nil
		case 'b':
			if rawValue, _, _err := decoder.source.ReadRune(); _err == nil {
				value = rawValue == '1'
				err = errors.New("Can not read boolean value")
			} else {
				err = errors.New("Can not read boolean value")
			}

			decoder.expect(VALUES_SEPARATOR)
		case 'i':
			if rawValue, _err := decoder.readUntil(VALUES_SEPARATOR); _err == nil {
				if value, _err = strconv.Atoi(rawValue); _err != nil {
					err = fmt.Errorf("Can not convert %v to Int:%v", rawValue, _err)
				}
			} else {
				err = errors.New("Can not read int value")
			}
		case 'd':
			if rawValue, _err := decoder.readUntil(VALUES_SEPARATOR); _err == nil {
				if value, _err = strconv.ParseFloat(rawValue, 64); _err != nil {
					err = fmt.Errorf("Can not convert %v to Float:%v", rawValue, _err)
				}
			} else {
				err = errors.New("Can not read float value")
			}
		case 's':
			value, err = decoder.decodeString()
			decoder.expect(VALUES_SEPARATOR)
		case 'a':
			value, err = decoder.decodeArray()
			decoder.allow(VALUES_SEPARATOR)
		case 'O':
			value, err = decoder.decodeObject()
		case 'C':
			value, err = decoder.decodeSerializableObject()
		default:
			log.Fatalf("Undefined token: %v [%#U]", token, token)
		}
	}
	return value, err<commit_msg>Panic instead of fatal on unknown token<commit_after>package php_session_decoder

import (
	"bytes"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"log"
)

type PhpDecoder struct {
	DecodeFunc	SerializableDecodeFunc
	RawData		string
	source		*strings.Reader
	data		PhpSessionData
}

func NewPhpDecoder(phpSession string) *PhpDecoder {
	sessionData := make(PhpSessionData)
	d := &PhpDecoder{
		RawData:	phpSession,
		source:		strings.NewReader(phpSession),
		data:		sessionData,
	}
	return d
}

func (decoder *PhpDecoder) Decode() (PhpSessionData, error) {
	var resultErr error
	for {
		if valueName, err := decoder.readUntil(VALUE_NAME_SEPARATOR); err == nil {
			if value, err := decoder.DecodeValue(); err == nil {
				decoder.data[valueName] = value
			} else {
				resultErr = fmt.Errorf("Can not read variable(%v) value:%v", valueName, err)
				break
			}
		} else {
			break
		}
	}
	return decoder.data, resultErr
}

func (decoder *PhpDecoder) DecodeValue() (PhpValue, error) {
	var (
		value PhpValue
		err   error
	)

	if token, _, err := decoder.source.ReadRune(); err == nil {
		decoder.expect(TYPE_VALUE_SEPARATOR)
		switch token {
		case 'N':
			value = nil
		case 'b':
			if rawValue, _, _err := decoder.source.ReadRune(); _err == nil {
				value = rawValue == '1'
				err = errors.New("Can not read boolean value")
			} else {
				err = errors.New("Can not read boolean value")
			}

			decoder.expect(VALUES_SEPARATOR)
		case 'i':
			if rawValue, _err := decoder.readUntil(VALUES_SEPARATOR); _err == nil {
				if value, _err = strconv.Atoi(rawValue); _err != nil {
					err = fmt.Errorf("Can not convert %v to Int:%v", rawValue, _err)
				}
			} else {
				err = errors.New("Can not read int value")
			}
		case 'd':
			if rawValue, _err := decoder.readUntil(VALUES_SEPARATOR); _err == nil {
				if value, _err = strconv.ParseFloat(rawValue, 64); _err != nil {
					err = fmt.Errorf("Can not convert %v to Float:%v", rawValue, _err)
				}
			} else {
				err = errors.New("Can not read float value")
			}
		case 's':
			value, err = decoder.decodeString()
			decoder.expect(VALUES_SEPARATOR)
		case 'a':
			value, err = decoder.decodeArray()
			decoder.allow(VALUES_SEPARATOR)
		case 'O':
			value, err = decoder.decodeObject()
		case 'C':
			value, err = decoder.decodeSerializableObject()
		default:
			log.Panicf("Undefined token: %v [%#U]", token, token)
		}
	}
	return value, err
{'fraction_non_alphanumeric': 0.13114754098360656, 'fraction_numerical': 0.0012610340479192938, 'mean_word_length': 3.510900473933649, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7000724', 'n_tokens_mistral': 1952, 'n_tokens_neox': 1688, 'n_words': 439}