<commit_before>package parser

import (
	"fmt"
	"io"
	"regexp"
	"strings"
)

type Match interface{}

type MatchString string

type MatchTree []Match

type TaggedMatch struct {
	Match Match
	Tag   string
}

type Grammar struct {
	parse func(rs io.ReadSeeker) (Match, error)
}

func (g *Grammar) Set(ng *Grammar) {
	g.parse = ng.parse
}

func (g *Grammar) Node(node func(Match) (Match, error)) {
	oldp := g.parse
	g.parse = func(rs io.ReadSeeker) (Match, error) {
		m, err := oldp(rs)
		if err == nil {
			m, err = node(m)
			if err == nil {
				return m, nil
			}
		}
		return nil, err
	}
}

func (g *Grammar) Parse(rs io.ReadSeeker) (Match, error) {
	m, err := g.parse(rs)
	if err != nil {
		return nil, err
	}
	return m, nil
}

func (g *Grammar) ParseString(s string) (Match, error) {
	m, err := g.parse(strings.NewReader(s))
	if err != nil {
		return nil, err
	}
	return m, nil
}

func Set(set string) *Grammar {
	regset, _ := regexp.Compile(fmt.Sprintf("[%s]", set))
	return &Grammar{parse: func(rs io.ReadSeeker) (Match, error) {
		pos, _ := rs.Seek(0, 1)
		b := make([]byte, 1)
		c, _ := rs.Read(b)
		if c < 1 {
			rs.Seek(pos, 0)
			return nil, fmt.Errorf("Unexpected EOF")
		}
		if regset.Match(b) {
			m := MatchString(b)
			return m, nil
		}
		rs.Seek(pos, 0)
		return nil, fmt.Errorf("Expected %s, got %s", set, string(b))
	}}
}

func Range(c1 ) *Grammar {
	regset, _ := regexp.Compile(fmt.Sprintf("[%s]", set))
	return &Grammar{parse: func(rs io.ReadSeeker) (Match, error) {
		pos, _ := rs.Seek(0, 1)
		b := make([]byte, 1)
		c, _ := rs.Read(b)
		if c < 1 {
			rs.Seek(pos, 0)
			return nil, fmt.Errorf("Unexpected EOF")
		}
		if regset.Match(b) {
			m := MatchString(b)
			return m, nil
		}
		rs.Seek(pos, 0)
		return nil, fmt.Errorf("Expected %s, got %s", set, string(b))<commit_msg>remove development code<commit_after>package parser

import (
	"fmt"
	"io"
	"regexp"
	"strings"
)

type Match interface{}

type MatchString string

type MatchTree []Match

type TaggedMatch struct {
	Match Match
	Tag   string
}

type Grammar struct {
	parse func(rs io.ReadSeeker) (Match, error)
}

func (g *Grammar) Set(ng *Grammar) {
	g.parse = ng.parse
}

func (g *Grammar) Node(node func(Match) (Match, error)) {
	oldp := g.parse
	g.parse = func(rs io.ReadSeeker) (Match, error) {
		m, err := oldp(rs)
		if err == nil {
			m, err = node(m)
			if err == nil {
				return m, nil
			}
		}
		return nil, err
	}
}

func (g *Grammar) Parse(rs io.ReadSeeker) (Match, error) {
	m, err := g.parse(rs)
	if err != nil {
		return nil, err
	}
	return m, nil
}

func (g *Grammar) ParseString(s string) (Match, error) {
	m, err := g.parse(strings.NewReader(s))
	if err != nil {
		return nil, err
	}
	return m, nil
}

func Set(set string) *Grammar {
	regset, _ := regexp.Compile(fmt.Sprintf("[%s]", set))
	return &Grammar{parse: func(rs io.ReadSeeker) (Match, error) {
		pos, _ := rs.Seek(0, 1)
		b := make([]byte, 1)
		c, _ := rs.Read(b)
		if c < 1 {
			rs.Seek(pos, 0)
			return nil, fmt.Errorf("Unexpected EOF")
		}
		if regset.Match(b) {
			m := MatchString(b)
			return m, nil
		}
		rs.Seek(pos, 0)
		return nil, fmt.Errorf("Expected %s, got %s", set, string(b))
{'fraction_non_alphanumeric': 0.16825396825396827, 'fraction_numerical': 0.006031746031746032, 'mean_word_length': 3.275440976933514, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6828644', 'n_tokens_mistral': 1425, 'n_tokens_neox': 1319, 'n_words': 357}