<commit_before>package bibtex

import (
	"errors"
	"fmt"
)

type parser struct {
	lexer     *lexer
	lexemes   chan lexeme
	lastToken lexeme
}

// @todo move to Reader

func ParseBibTeXEntry(in string) (BibTeXEntry, error) {
	return parseBibTeXSingle(in)
}

func ParseBibTeXEntries(in string) []BibTeXEntry {
	return parseBibTeXMultiple(in)
}

func parseBibTeXSingle(in string) (BibTeXEntry, error) {
	lexer, lexemes := lexBibTeX(in)
	p := &parser{
		lexer:   lexer,
		lexemes: lexemes,
	}
	return p.parse()
}

func parseBibTeXMultiple(in string) (results []BibTeXEntry) {
	results = make([]BibTeXEntry, 0)

	lexer, lexemes := lexBibTeX(in)
	p := &parser{
		lexer:   lexer,
		lexemes: lexemes,
	}

	for p.lastToken.typ != tokenEOF {
		entry, err := p.parse()
		if err == nil {
			results = append(results, entry)
		}

	}
	return results
}

func (p *parser) parse() (b BibTeXEntry, err error) {
	b = newBibTeXEntry()
	err = p.expect(tokenEntryStart)

	if err != nil {
		return b, err
	}
	b, err = p.entry_type(b)

	err = p.expect(tokenLeftBrace)
	if err != nil {
		return b, err
	}

	b, err = p.entry_body(b)

	return b, err
}

func (p *parser) entry_type(b BibTeXEntry) (r BibTeXEntry, err error) {
	r = b
	err = p.expect(tokenIdentifier)
	r.Type = p.lastToken.val
	return r, err
}

func (p *parser) entry_body(b BibTeXEntry) (r BibTeXEntry, err error) {
	r = b
	err = p.expect(tokenIdentifier)
	if err != nil {
		return r, err
	}
	r.Identifier = p.lastToken.val

	return p.value_list(r)
}

func (p *parser) value_list(b BibTeXEntry) (r BibTeXEntry, err error) {
	r = b

	t := p.lastToken
	if t.typ == tokenRightBrace {
		return r, nil
	}
	if t.typ == tokenError || t.typ == tokenEOF {
		return r, errors.New(fmt.Sprint(t))
	}

	ok := p.accept(tokenIdentifier)
	if ok {<commit_msg>Add accept of comma after entry identifier<commit_after>package bibtex

import (
	"errors"
	"fmt"
)

type parser struct {
	lexer     *lexer
	lexemes   chan lexeme
	lastToken lexeme
}

// @todo move to Reader

func ParseBibTeXEntry(in string) (BibTeXEntry, error) {
	return parseBibTeXSingle(in)
}

func ParseBibTeXEntries(in string) []BibTeXEntry {
	return parseBibTeXMultiple(in)
}

func parseBibTeXSingle(in string) (BibTeXEntry, error) {
	lexer, lexemes := lexBibTeX(in)
	p := &parser{
		lexer:   lexer,
		lexemes: lexemes,
	}
	return p.parse()
}

func parseBibTeXMultiple(in string) (results []BibTeXEntry) {
	results = make([]BibTeXEntry, 0)

	lexer, lexemes := lexBibTeX(in)
	p := &parser{
		lexer:   lexer,
		lexemes: lexemes,
	}

	for p.lastToken.typ != tokenEOF {
		entry, err := p.parse()
		if err == nil {
			results = append(results, entry)
		}

	}
	return results
}

func (p *parser) parse() (b BibTeXEntry, err error) {
	b = newBibTeXEntry()
	err = p.expect(tokenEntryStart)

	if err != nil {
		return b, err
	}
	b, err = p.entry_type(b)

	err = p.expect(tokenLeftBrace)
	if err != nil {
		return b, err
	}

	b, err = p.entry_body(b)

	return b, err
}

func (p *parser) entry_type(b BibTeXEntry) (r BibTeXEntry, err error) {
	r = b
	err = p.expect(tokenIdentifier)
	r.Type = p.lastToken.val
	return r, err
}

func (p *parser) entry_body(b BibTeXEntry) (r BibTeXEntry, err error) {
	r = b
	err = p.expect(tokenIdentifier)
	if err != nil {
		return r, err
	}
	r.Identifier = p.lastToken.val
	p.accept(tokenComma)

	return p.value_list(r)
}

func (p *parser) value_list(b BibTeXEntry) (r BibTeXEntry, err error) {
	r = b

	t := p.lastToken
	if t.typ == tokenRightBrace {
		return r, nil
	}
	if t.typ == tokenError || t.typ == tokenEOF {
		return r, errors.New(fmt.Sprint(t))
	}

	ok := p.accept(tokenIdentifier)
	if ok {
{'fraction_non_alphanumeric': 0.1309886458044863, 'fraction_numerical': 0.0005538631957906397, 'mean_word_length': 3.6666666666666665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '269970', 'n_tokens_mistral': 1588, 'n_tokens_neox': 1430, 'n_words': 379}