<commit_before>package down

import (
	"errors"
	"fmt"
)

type Parser struct {
	source   string
	location int
	children []Noder
}

func (p Parser) Peek() (uint8, error) {
	if p.location+1 == len(p.source) {
		return ' ', errors.New("Cannot peek EOF")
	}

	return p.source[p.location+1], nil
}

func (p *Parser) Next() {
	p.location++
}

func (p Parser) End() bool {
	if p.location >= len(p.source) {
		return true
	}
	return false
}

func (p Parser) Back() {
	p.location--
}

func (p *Parser) parseLine() Noder {
	line := LineNode{RawTextNode{""}}
	for !p.End() {
		c := p.source[p.location]
		switch c {
		case uint8('#'):
			line.Child = p.parseHeader()
		case uint8('-'):
			line.Child = p.parseList()
		default:
			line.Child = ParagraphNode{Child: p.parseComposite()}
		}

		p.children = append(p.children, line)
		p.Next()
	}
	return LineNode{}
}

func (p *Parser) parseList() Noder {
	list := UnorderedListNode{}
	for !p.End() {
		c := p.source[p.location]

		if c != uint8('-') {
			return list
		}

		p.Next()
		list.AddChild(UnorderedListItemNode{p.parseListItem()})
		p.Next()
	}

	return list
}

func (p *Parser) parseListItem() Noder {
	if c := p.source[p.location]; c != uint8('-') && c != uint8(' ') {
		return p.parseComposite()
	}
	p.Next()
	return p.parseComposite()
}

func (p *Parser) parseRaw() RawTextNode {
	var content string

	for !p.End() {
		c := p.source[p.location]
		if c == uint8(10) {
			break
		}

		if c == uint8('*') || c == uint8('[') || c == uint8('-') {
			p.Back()
			break
		}

		content += string(p.source[p.location])
		p.Next()
	}

	return RawTextNode{Content: content}
}

func (p *Parser) parseItalics() Noder {
	var content string
	node := ItalicNode{}
	for !p.End() {
		c := p.source[p.location]
		if c == uint8(10) {
			break
		}

		if c == uint8('*') {
			node.Child.AddChild(RawTextNode{Content: content})
			return node
		}

		content += string(c)
		p.Next()
	}

	p.Back()
	return RawTextNode{"*" + content}
}

func (p *Parser) parseBold() Noder {
	var content string
	node := BoldStringNode{}
	if p.source[p.location] != uint8('*') {
		p.Back()
	} else {
		p.Next()
	}

	for !p.End() {
		c := p.source[p.location]

		if c == uint8(10) {
			break
		}

		if c == uint8('*') {
			peek, _ := p.Peek()
			p.Next()
			if peek != c {
				node.Child.AddChild(RawTextNode{Content: content})
				node.Child.AddChild(p.parseItalics())
				content = ""
				continue
			}
			p.Next()
			fmt.Println(string(peek), "is", string(c))

			node.Child.AddChild(RawTextNode{Content: content})
			return node
		}

		content += string(c)

		p.Next()
	}

	return RawTextNode{"**" + content}
}

func (p *Parser) parseComposite() CompositeStringNode {
	composite := CompositeStringNode{}

	for !p.End() {
		c := p.source[p.location]
		if c == uint8(10) {
			break
		}

		switch c {
		case uint8('*'):
			peek, _ := p.Peek()
			p.Next()
			if peek == c {
				p.Next()
				composite.AddChild(p.parseBold())
				continue
			}

			composite.AddChild(p.parseItalics())
		case uint8('['):
			p.Next()
			composite.AddChild(p.parseLink())
		default:
			composite.AddChild(p.parseRaw())
		}

		if c == uint8(']') || c == uint8(')') || c == uint8('-') {
			break
		}

		p.Next()
	}
	return composite
}

func (p *Parser) parseLink() Noder {
	link := LinkNode{}
	var place string

	for !p.End() {
		c := p.source[p.location]
		switch c {
		case ']':
			link.Text = RawTextNode{place}
		case '(':
			place = ""
			p.Next()
			continue
		case ')':
			link.Addr = RawTextNode{place}
			return link<commit_msg>parser: fix over extending backwards after composite<commit_after>package down

import (
	"errors"
	"fmt"
)

type Parser struct {
	source   string
	location int
	children []Noder
}

func (p Parser) Peek() (uint8, error) {
	if p.location+1 == len(p.source) {
		return ' ', errors.New("Cannot peek EOF")
	}

	return p.source[p.location+1], nil
}

func (p *Parser) Next() {
	p.location++
}

func (p Parser) End() bool {
	if p.location >= len(p.source) {
		return true
	}
	return false
}

func (p Parser) Back() {
	p.location--
}

func (p *Parser) parseLine() Noder {
	line := LineNode{RawTextNode{""}}
	for !p.End() {
		c := p.source[p.location]
		switch c {
		case uint8('#'):
			line.Child = p.parseHeader()
		case uint8('-'):
			line.Child = p.parseList()
		default:
			line.Child = ParagraphNode{Child: p.parseComposite()}
		}

		p.children = append(p.children, line)
		p.Next()
	}
	return LineNode{}
}

func (p *Parser) parseList() Noder {
	list := UnorderedListNode{}
	for !p.End() {
		c := p.source[p.location]

		if c != uint8('-') {
			fmt.Println("FINISHED LIST WITH", string(c), c, p.source[:p.location])
			return list
		}

		p.Next()
		list.AddChild(UnorderedListItemNode{p.parseListItem()})
	}

	return list
}

func (p *Parser) parseListItem() Noder {
	if c := p.source[p.location]; c != uint8('-') && c != uint8(' ') {
		return p.parseComposite()
	}
	p.Next()
	return p.parseComposite()
}

func (p *Parser) parseRaw() RawTextNode {
	var content string

	for !p.End() {
		c := p.source[p.location]
		if c == uint8(10) {
			break
		}

		if c == uint8('*') || c == uint8('[') || c == uint8('-') {
			p.Back()
			break
		}

		content += string(p.source[p.location])
		p.Next()
	}

	return RawTextNode{Content: content}
}

func (p *Parser) parseItalics() Noder {
	var content string
	node := ItalicNode{}
	for !p.End() {
		c := p.source[p.location]
		if c == uint8(10) {
			break
		}

		if c == uint8('*') {
			node.Child.AddChild(RawTextNode{Content: content})
			return node
		}

		content += string(c)
		p.Next()
	}

	p.Back()
	return RawTextNode{"*" + content}
}

func (p *Parser) parseBold() Noder {
	var content string
	node := BoldStringNode{}
	if p.source[p.location] != uint8('*') {
		p.Back()
	} else {
		p.Next()
	}

	for !p.End() {
		c := p.source[p.location]

		if c == uint8(10) {
			break
		}

		if c == uint8('*') {
			peek, _ := p.Peek()
			p.Next()
			if peek != c {
				node.Child.AddChild(RawTextNode{Content: content})
				node.Child.AddChild(p.parseItalics())
				content = ""
				continue
			}
			p.Next()
			fmt.Println(string(peek), "is", string(c))

			node.Child.AddChild(RawTextNode{Content: content})
			return node
		}

		content += string(c)

		p.Next()
	}

	return RawTextNode{"**" + content}
}

func (p *Parser) parseComposite() CompositeStringNode {
	composite := CompositeStringNode{}

	for !p.End() {
		c := p.source[p.location]
		if c == uint8(10) {
			break
		}

		switch c {
		case uint8('*'):
			peek, _ := p.Peek()
			p.Next()
			if peek == c {
				p.Next()
				composite.AddChild(p.parseBold())
				continue
			}

			composite.AddChild(p.parseItalics())
		case uint8('['):
			p.Next()
			composite.AddChild(p.parseLink())
		default:
			composite.AddChild(p.parseRaw())
		}

		if c == uint8(']') || c == uint8(')') || c == uint8('-') {

			break
		}

		p.Next()
	}
	return composite
}

func (p *Parser) parseLink() Noder {
	link := LinkNode{}
	var place string

	for !p.End() {
		c := p.source[p.location]
		switch c {
		case ']':
			link.Text = RawTextNode{place}
		case '(':
			place = ""
			p.Next()
			continue
		case ')':
			link.Addr = RawTextNode{place}
			return link
{'fraction_non_alphanumeric': 0.1849363012739745, 'fraction_numerical': 0.008679826403471931, 'mean_word_length': 3.0179977502812148, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1582401', 'n_tokens_mistral': 3192, 'n_tokens_neox': 2830, 'n_words': 630}