<commit_before>// Package strparse is a simple helper package for parsing strings
package strparse

import (
	"strings"
	"unicode/utf8"
)

type Parser struct {
	Str        string
	pos, width int
}

func (p *Parser) next() rune {
	if p.pos == len(p.Str) {
		p.width = 0
		return -1
	}
	r, s := utf8.DecodeRuneInString(p.Str)
	p.pos += s
	p.width = s
	return r
}

func (p *Parser) backup() {
	if p.width > 0 {
		p.pos -= p.width
		p.width = 0
	}
}

func (p *Parser) Peek() rune {
	r := p.next()
	p.backup()
	return r
}<commit_msg>added New func<commit_after>// Package strparse is a simple helper package for parsing strings
package strparse

import (
	"strings"
	"unicode/utf8"
)

type Parser struct {
	Str        string
	pos, width int
}

func New(s string) *Parser {
	return &Parser{Str: s}
}

func (p *Parser) next() rune {
	if p.pos == len(p.Str) {
		p.width = 0
		return -1
	}
	r, s := utf8.DecodeRuneInString(p.Str)
	p.pos += s
	p.width = s
	return r
}

func (p *Parser) backup() {
	if p.width > 0 {
		p.pos -= p.width
		p.width = 0
	}
}

func (p *Parser) Peek() rune {
	r := p.next()
	p.backup()
	return r
}
{'fraction_non_alphanumeric': 0.14645103324348607, 'fraction_numerical': 0.01078167115902965, 'mean_word_length': 2.7635135135135136, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3566305', 'n_tokens_mistral': 482, 'n_tokens_neox': 462, 'n_words': 141}