<commit_before>// SPDX-License-Identifier: MIT

package lexer

import (
	"testing"

	"github.com/issue9/assert"

	"github.com/caixw/apidoc/v6/core"
)

func TestNewLexer(t *testing.T) {
	a := assert.New(t)

	l, err := New([]byte("// doc"))
	a.NotError(err).NotNil(l)

	// 中间三个字节表示一个汉字
	l, err = New([]byte{0b01001001, 0b11100111, 0b10011111, 0b10100101, 0b01110100})
	a.NotError(err).NotNil(l)

	// 中间三个字节表示一个汉字，编码错误
	l, err = New([]byte{0b01001001, 0b01100111, 0b10011111, 0b10100101, 0b01110100})
	a.Error(err).Nil(l)
}

func TestLexer_Position(t *testing.T) {
	a := assert.New(t)

	l, err := New([]byte("l0\nl1\nl2\nl3\n"))
	a.NotError(err).NotNil(l)

	l.Next(3)
	a.Equal(l.Position(), Position{
		Position: core.Position{Line: 1, Character: 0},
		Offset:   3,
	})

	l.Next(4)
	a.Equal(l.Position(), Position{
		Position: core.Position{Line: 2, Character: 1},
		Offset:   7,
	})

	l.Next(3)
	l.Next(3)
	a.Equal(l.Position(), Position{
		Position: core.Position{Line: 4, Character: 0},
		Offset:   12,
	})

	l, err = New([]byte("12中文ab"))
	a.NotError(err).NotNil(l)
	l.Next(2)
	a.Equal(l.Position(), Position{
		Position: core.Position{Line: 0, Character: 2},
		Offset:   2,
	})

	l.Next(2)
	a.Equal(l.Position(), Position{
		Position: core.Position{Line: 0, Character: 4},
		Offset:   8,
	})

	l.Next(10)
	a.True(l.AtEOF())

	p := Position{
		Position: core.Position{Line: 0, Character: 2},
		Offset:   2,
	}
	l.SetPosition(p)
	a.False(l.AtEOF()).
		Equal(0, l.prev.Offset).
		Equal(l.current, p)
}

func TestLexer_Match(t *testing.T) {
	a := assert.New(t)

	l := &Lexer{
		data: []byte("ab中\ncd"),
	}

	a.False(l.Match("")).Equal(0, l.current.Offset)
	a.False(l.Match("b")).Equal(0, l.current.Offset)
	a.True(l.Match("ab")).Equal(2, l.current.Offset)
	a.False(l.Match("ab")).Equal(2, l.current.Offset)
	a.True(l.Match("中")).Equal(l.current, Position{
		Position: core.Position{Line: 0, Character: 3},
		Offset:   5,
	})

	l.Rollback()
	l.Rollback()
	l.Rollback()
	a.True(l.Match("中")).Equal(l.current, Position{
		Position: core.Position{Line: 0, Character: 3},
		Offset:   5,
	})

	l.Next(len(l.data))
	a.False(l.Match("ab")).Equal(l.current, Position{
		Position: core.Position{Line: 1, Character: 2},
		Offset:   8,
	})
}

func TestLexer_Spaces(t *testing.T) {
	a := assert.New(t)
	l, err := New([]byte("    0 \n  1 "))
	a.NotError(err).NotNil(l)

	l.Spaces('\n')
	a.Equal(string(l.Next(1)), "0")

	// 无法跳过换行符
	l.Next(1)
	l.Spaces('\n')
	l.Spaces('\n')
	l.Spaces('\n')
	l.Spaces('\n')
	a.Empty(l.Spaces('\n'))
	a.Equal(string(l.Next(1)), "\n")

	l.Next(1)
	a.Equal(1, len(l.Spaces('\n')))
	l.Rollback()
	a.Equal(1, len(l.Spaces('\n')))
	a.Equal(string(l.Next(1)), "1")

	l.Next(1)
	l.Spaces('\n')
	l.Spaces('\n')
	a.Equal(l.current.Offset, len(l.data))
}

func TestLexer_Delim(t *testing.T) {
	a := assert.New(t)

	l, err := New([]byte("123"))
	a.NotError(err).NotNil(l)
	a.Nil(l.Delim('\n', true))
	a.Nil(l.Delim(0, true))

	l, err = New([]byte("123\n"))
	a.NotError(err).NotNil(l)
	a.Equal(string(l.Delim('\n', true)), "123\n").
		Equal(l.current.Offset, 4)

	l = &Lexer{data: []byte("123\n"), current: Position{Offset: 1}}
	a.Equal(string(l.Delim('\n', true)), "23\n").
		Equal(l.current.Offset, 4)<commit_msg>test(internal/lexer): 添加对 Lexer.Spaces 的测试用例<commit_after>// SPDX-License-Identifier: MIT

package lexer

import (
	"testing"

	"github.com/issue9/assert"

	"github.com/caixw/apidoc/v6/core"
)

func TestNewLexer(t *testing.T) {
	a := assert.New(t)

	l, err := New([]byte("// doc"))
	a.NotError(err).NotNil(l)

	// 中间三个字节表示一个汉字
	l, err = New([]byte{0b01001001, 0b11100111, 0b10011111, 0b10100101, 0b01110100})
	a.NotError(err).NotNil(l)

	// 中间三个字节表示一个汉字，编码错误
	l, err = New([]byte{0b01001001, 0b01100111, 0b10011111, 0b10100101, 0b01110100})
	a.Error(err).Nil(l)
}

func TestLexer_Position(t *testing.T) {
	a := assert.New(t)

	l, err := New([]byte("l0\nl1\nl2\nl3\n"))
	a.NotError(err).NotNil(l)

	l.Next(3)
	a.Equal(l.Position(), Position{
		Position: core.Position{Line: 1, Character: 0},
		Offset:   3,
	})

	l.Next(4)
	a.Equal(l.Position(), Position{
		Position: core.Position{Line: 2, Character: 1},
		Offset:   7,
	})

	l.Next(3)
	l.Next(3)
	a.Equal(l.Position(), Position{
		Position: core.Position{Line: 4, Character: 0},
		Offset:   12,
	})

	l, err = New([]byte("12中文ab"))
	a.NotError(err).NotNil(l)
	l.Next(2)
	a.Equal(l.Position(), Position{
		Position: core.Position{Line: 0, Character: 2},
		Offset:   2,
	})

	l.Next(2)
	a.Equal(l.Position(), Position{
		Position: core.Position{Line: 0, Character: 4},
		Offset:   8,
	})

	l.Next(10)
	a.True(l.AtEOF())

	p := Position{
		Position: core.Position{Line: 0, Character: 2},
		Offset:   2,
	}
	l.SetPosition(p)
	a.False(l.AtEOF()).
		Equal(0, l.prev.Offset).
		Equal(l.current, p)
}

func TestLexer_Match(t *testing.T) {
	a := assert.New(t)

	l := &Lexer{
		data: []byte("ab中\ncd"),
	}

	a.False(l.Match("")).Equal(0, l.current.Offset)
	a.False(l.Match("b")).Equal(0, l.current.Offset)
	a.True(l.Match("ab")).Equal(2, l.current.Offset)
	a.False(l.Match("ab")).Equal(2, l.current.Offset)
	a.True(l.Match("中")).Equal(l.current, Position{
		Position: core.Position{Line: 0, Character: 3},
		Offset:   5,
	})

	l.Rollback()
	l.Rollback()
	l.Rollback()
	a.True(l.Match("中")).Equal(l.current, Position{
		Position: core.Position{Line: 0, Character: 3},
		Offset:   5,
	})

	l.Next(len(l.data))
	a.False(l.Match("ab")).Equal(l.current, Position{
		Position: core.Position{Line: 1, Character: 2},
		Offset:   8,
	})
}

func TestLexer_Spaces(t *testing.T) {
	a := assert.New(t)
	l, err := New([]byte("    0 \n  1 "))
	a.NotError(err).NotNil(l)

	l.Spaces('\n')
	a.Equal(string(l.Next(1)), "0")

	// 无法跳过换行符
	l.Next(1)
	l.Spaces('\n')
	l.Spaces('\n')
	l.Spaces('\n')
	l.Spaces('\n')
	a.Empty(l.Spaces('\n'))
	a.Equal(string(l.Next(1)), "\n")

	a.Equal(l.Spaces('\t'), "  ")
	l.Rollback()
	a.Equal(2, len(l.Spaces(0)))

	l.Rollback()
	a.Equal(0, len(l.Spaces(' ')))
	a.Equal(2, len(l.Spaces(0)))

	a.Equal(string(l.Next(1)), "1")

	l.Next(1)
	l.Spaces('\n')
	l.Spaces('\n')
	a.Equal(l.current.Offset, len(l.data))
}

func TestLexer_Delim(t *testing.T) {
	a := assert.New(t)

	l, err := New([]byte("123"))
	a.NotError(err).NotNil(l)
	a.Nil(l.Delim('\n', true))
	a.Nil(l.Delim(0, true))

	l, err = New([]byte("123\n"))
	a.NotError(err).NotNil(l)
	a.Equal(string(l.Delim('\n', true)), "123\n").
		Equal(l.current.Offset, 4)

	l = &Lexer{data: []byte("123\n"), current: Position{Offset: 1}}
	a.Equal(string(l.Delim('\n', true)), "23\n").
		Equal(l.current.Offset, 4)
{'fraction_non_alphanumeric': 0.2339611085591793, 'fraction_numerical': 0.05205940897259225, 'mean_word_length': 3.9710806697108065, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6117289', 'n_tokens_mistral': 3328, 'n_tokens_neox': 3074, 'n_words': 477}