<commit_before>package parsex

import (
	"errors"
	"fmt"
	"io"
	"reflect"
	"strconv"
	"time"
	"unicode"
)

type NotEqual struct {
	Except interface{}
	Value  interface{}
	Pos    int
}

func (this NotEqual) Error() string {
	return fmt.Sprintf("Postion: %d,  Except %v but got %v", this.Pos, this.Except, this.Value)
}

type TypeError struct {
	Type  string
	Value interface{}
	Pos   int
}

func (this TypeError) Error() string {
	return fmt.Sprintf("Postion: %d,  Except %v as a %s", this.Pos, this.Value, this.Type)
}

func equals(x interface{}) func(int, interface{}) (interface{}, error) {
	return func(pos int, data interface{}) (interface{}, error) {
		if reflect.DeepEqual(x, data) {
			return x, nil
		} else {
			return data, NotEqual{x, data, pos}
		}
	}
}

func always(pos int, x interface{}) (interface{}, error) {
	return x, nil
}

func Rune(r rune) Parser {
	return func(st ParsexState) (interface{}, error) {
		ru, err := st.Next(equals(r))
		if err != nil {
			return nil, err
		} else {
			return ru, nil
		}
	}
}

// match anyone else a eof or panic
func AnyOne(st ParsexState) (interface{}, error) {
	return st.Next(always)
}
func TheOne(one interface{}) Parser {
	return func(st ParsexState) (interface{}, error) {
		_, err := st.Next(equals(one))
		if err == nil {
			return one, nil
		} else {
			return nil, err
		}
	}
}

func Eof(st ParsexState) (interface{}, error) {
	r, err := st.Next(always)
	if err == nil {
		return nil, st.Trap("Except EOF but got %c", r)
	} else {
		if err == io.EOF {
			return nil, nil
		} else {
			return nil, err
		}
	}
}

// parsex 的 String 尝试匹配 State 的下一个 Token，这与 parsec 不同
func String(s string) Parser {
	return func(st ParsexState) (interface{}, error) {
		pos := st.Pos()

		// try and match string
		_, err := st.Next(equals(s))
		if err != nil {
			st.SeekTo(pos)
			return nil, err
		}
		return s, nil
	}
}

func AnyRune(st ParsexState) (interface{}, error) {
	c, err := st.Next(always)

	if err == nil {
		return c, nil
	} else {
		if err == io.EOF {
			return nil, st.Trap("Unexpected end of file")
		} else {
			return nil, err
		}
	}
}
func canbeInt(pos int, x interface{}) (interface{}, error) {
	switch val := x.(type) {
	case int:
		return val, nil
	case int64:
		return int(val), nil
	case int32:
		return int(val), nil
	case float64:
		return int(val), nil
	case float32:
		return int(val), nil
	case string:
		return strconv.Atoi(val)
	case complex64:
		if imag(val) == 0 {
			return int(real(val)), nil<commit_msg>publish parsex always as Always<commit_after>package parsex

import (
	"errors"
	"fmt"
	"io"
	"reflect"
	"strconv"
	"time"
	"unicode"
)

type NotEqual struct {
	Except interface{}
	Value  interface{}
	Pos    int
}

func (this NotEqual) Error() string {
	return fmt.Sprintf("Postion: %d,  Except %v but got %v", this.Pos, this.Except, this.Value)
}

type TypeError struct {
	Type  string
	Value interface{}
	Pos   int
}

func (this TypeError) Error() string {
	return fmt.Sprintf("Postion: %d,  Except %v as a %s", this.Pos, this.Value, this.Type)
}

func equals(x interface{}) func(int, interface{}) (interface{}, error) {
	return func(pos int, data interface{}) (interface{}, error) {
		if reflect.DeepEqual(x, data) {
			return x, nil
		} else {
			return data, NotEqual{x, data, pos}
		}
	}
}

func Always(pos int, x interface{}) (interface{}, error) {
	return x, nil
}

func Rune(r rune) Parser {
	return func(st ParsexState) (interface{}, error) {
		ru, err := st.Next(equals(r))
		if err != nil {
			return nil, err
		} else {
			return ru, nil
		}
	}
}

// match anyone else a eof or panic
func AnyOne(st ParsexState) (interface{}, error) {
	return st.Next(Always)
}
func TheOne(one interface{}) Parser {
	return func(st ParsexState) (interface{}, error) {
		_, err := st.Next(equals(one))
		if err == nil {
			return one, nil
		} else {
			return nil, err
		}
	}
}

func Eof(st ParsexState) (interface{}, error) {
	r, err := st.Next(Always)
	if err == nil {
		return nil, st.Trap("Except EOF but got %c", r)
	} else {
		if err == io.EOF {
			return nil, nil
		} else {
			return nil, err
		}
	}
}

// parsex 的 String 尝试匹配 State 的下一个 Token，这与 parsec 不同
func String(s string) Parser {
	return func(st ParsexState) (interface{}, error) {
		pos := st.Pos()

		// try and match string
		_, err := st.Next(equals(s))
		if err != nil {
			st.SeekTo(pos)
			return nil, err
		}
		return s, nil
	}
}

func AnyRune(st ParsexState) (interface{}, error) {
	c, err := st.Next(Always)

	if err == nil {
		return c, nil
	} else {
		if err == io.EOF {
			return nil, st.Trap("Unexpected end of file")
		} else {
			return nil, err
		}
	}
}
func canbeInt(pos int, x interface{}) (interface{}, error) {
	switch val := x.(type) {
	case int:
		return val, nil
	case int64:
		return int(val), nil
	case int32:
		return int(val), nil
	case float64:
		return int(val), nil
	case float32:
		return int(val), nil
	case string:
		return strconv.Atoi(val)
	case complex64:
		if imag(val) == 0 {
			return int(real(val)), nil
{'fraction_non_alphanumeric': 0.1466346153846154, 'fraction_numerical': 0.004407051282051282, 'mean_word_length': 3.3154710458081245, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1403037', 'n_tokens_mistral': 2015, 'n_tokens_neox': 1877, 'n_words': 591}