<commit_before>package pawn

import (
	"strings"
	"text/scanner"
)

type PGN struct {
	Tags map[string]string
}

func NewPGN() PGN {
	return PGN{
		Tags: map[string]string{},
	}
}

type PGNParser struct {
	pgn PGN
}

func NewPGNParser() PGNParser {
	return PGNParser{pgn: NewPGN()}
}

func (p PGNParser) ParseFromString(str string) PGN {
	reader := strings.NewReader(str)
	scanner := scanner.Scanner{}
	scanner.Init(reader)

	return p.ParseFromScanner(&scanner)
}

func (p PGNParser) ParseFromScanner(sc *scanner.Scanner) PGN {
	ParseTags(sc, &p.pgn)

	return p.pgn
}

func ParsePGN(str string) PGN {
	return NewPGNParser().ParseFromString(str)
}

func ParseTags(sc *scanner.Scanner, pgn *PGN) {
	scan := sc.Peek()

	for scan != scanner.EOF {
		switch scan {
		case '[', ']', '\n', '\r':
			scan = sc.Next()
		case '1': // First move; all tags have been read
			return
		default:
			sc.Scan()
			tag := sc.TokenText()
			sc.Scan()
			value := sc.TokenText()

			pgn.Tags[tag] = strings.Trim(value, "\"")
		}
		scan = sc.Peek()
	}
}

func ParseMoves(sc *scanner.Scanner, pgn *PGN) {
	sc.Mode = scanner.ScanIdents | scanner.ScanChars | scanner.ScanInts | scanner.ScanStrings
}<commit_msg>Verify CI behavior<commit_after>package pawn

import (
	"strings"
	"text/scanner"
)

type PGN struct {
	Tags map[string]string
}

func NewPGN() PGN {
	return PGN{
		Tags: map[string]string{},
	}
}

type PGNParser struct {
	pgn PGN
}

func NewPGNParser() PGNParser {
	return PGNParser{pgn: NewPGN()}
}

func (p PGNParser) ParseFromString(str string) PGN {
	reader := strings.NewReader(str)
	scanner := scanner.Scanner{}
	scanner.Init(reader)

	return p.ParseFromScanner(&scanner)
}

func (p PGNParser) ParseFromScanner(sc *scanner.Scanner) PGN {
	ParseTags(sc, &p.pgn)

	return p.pgn
}

func ParsePGN(str string) PGN {
	return NewPGNParser().ParseFromString(str)
}

func ParseTags(sc *scanner.Scanner, pgn *PGN) {
	scan := sc.Peek()

	for scan != scanner.EOF {
		switch scan {
		case '[', ']', '\n', '\r':
			scan = sc.Next()
		case '1': // First move; all tags have been read
			return
		default:
			sc.Scan()
			tag := sc.TokenText()
			sc.Scan()
			value := sc.TokenText()

			pgn.Tags[tag] = strings.Trim(value, "\"")
		}
		scan = sc.Peek()
	}
}

func ParseMoves(sc *scanner.Scanner, pgn *PGN) {
	// sc.Mode = scanner.ScanIdents | scanner.ScanChars | scanner.ScanInts | scanner.ScanStrings
}
{'fraction_non_alphanumeric': 0.15126050420168066, 'fraction_numerical': 0.0008403361344537816, 'mean_word_length': 3.8198380566801617, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1572979', 'n_tokens_mistral': 1005, 'n_tokens_neox': 933, 'n_words': 212}