<commit_before>package mark

import (
	"fmt"
	"regexp"
	"unicode/utf8"
)

// type position
type Pos int

// itemType identifies the type of lex items.
type itemType int

// Item represent a token or text string returned from the scanner
type item struct {
	typ itemType // The type of this item.
	pos Pos      // The starting position, in bytes, of this item in the input string.
	val string   // The value of this item.
}

const eof = -1 // Zero value so closed channel delivers EOF

const (
	itemError itemType = iota // Error occurred; value is text of error
	// Intersting things
	itemNewLine
	itemHTML
	// Block Elements
	itemText
	itemLineBreak
	itemHeading
	itemLHeading
	itemBlockQuote
	itemList
	itemCodeBlock
	itemGfmCodeBlock
	itemHr
	itemTable
	// Span Elements
	itemLinks
	itemStrong
	itemItalic
	itemStrike
	itemCode
	itemImages
	itemBr
)

var (
	reEmphasise = "^_{%[1]d}([\\s\\S]+?)_{%[1]d}|^\\*{%[1]d}([\\s\\S]+?)\\*{%[1]d}"
	reGfmCode   = "^%s{3,} *(\\S+)? *\n([\\s\\S]+?)\\s*%s{3,}$*(?:\n+|$)"
)

// Block Grammer
var block = map[itemType]*regexp.Regexp{
	itemHeading:   regexp.MustCompile("^ *(#{1,6}) *([^\n]+?) *#* *(?:\n+|$)"),
	itemHr:        regexp.MustCompile("^( *[-*_]){3,} *(?:\n+|$)"),
	itemCodeBlock: regexp.MustCompile("^( {4}[^\n]+\n*)+"),
	// Backreferences is unavailable
	itemGfmCodeBlock: regexp.MustCompile(fmt.Sprintf(reGfmCode, "`", "`") + "|" + fmt.Sprintf(reGfmCode, "~", "~")),
	itemList:         regexp.MustCompile("(?:[*+-]|\\d+\\.)"),
}

// Inline Grammer
var span = map[itemType]*regexp.Regexp{
	itemItalic: regexp.MustCompile(fmt.Sprintf(reEmphasise, 1)),
	itemStrong: regexp.MustCompile(fmt.Sprintf(reEmphasise, 2)),
	itemStrike: regexp.MustCompile("^~{2}([\\s\\S]+?)~{2}"),
	// itemMixed(e.g: ***str***, ~~*str*~~) will be part of the parser
	itemCode: regexp.MustCompile("^`{1,2}\\s*([\\s\\S]*?[^`])\\s*`{1,2}"),
}<commit_msg>todo(lexer): add<commit_after>package mark

import (
	"fmt"
	"regexp"
	"unicode/utf8"
)

// type position
type Pos int

// itemType identifies the type of lex items.
type itemType int

// Item represent a token or text string returned from the scanner
type item struct {
	typ itemType // The type of this item.
	pos Pos      // The starting position, in bytes, of this item in the input string.
	val string   // The value of this item.
}

const eof = -1 // Zero value so closed channel delivers EOF

const (
	itemError itemType = iota // Error occurred; value is text of error
	// Intersting things
	itemNewLine
	itemHTML
	// Block Elements
	itemText
	itemLineBreak
	itemHeading
	itemLHeading
	itemBlockQuote
	itemList
	itemCodeBlock
	itemGfmCodeBlock
	itemHr
	itemTable
	// Span Elements
	itemLinks
	itemStrong
	itemItalic
	itemStrike
	itemCode
	itemImages
	itemBr
)

var (
	reEmphasise = "^_{%[1]d}([\\s\\S]+?)_{%[1]d}|^\\*{%[1]d}([\\s\\S]+?)\\*{%[1]d}"
	reGfmCode   = "^%s{3,} *(\\S+)? *\n([\\s\\S]+?)\\s*%s{3,}$*(?:\n+|$)"
)

// Block Grammer
var block = map[itemType]*regexp.Regexp{
	itemHeading:   regexp.MustCompile("^ *(#{1,6}) *([^\n]+?) *#* *(?:\n+|$)"),
	itemHr:        regexp.MustCompile("^( *[-*_]){3,} *(?:\n+|$)"),
	itemCodeBlock: regexp.MustCompile("^( {4}[^\n]+\n*)+"),
	// Backreferences is unavailable
	itemGfmCodeBlock: regexp.MustCompile(fmt.Sprintf(reGfmCode, "`", "`") + "|" + fmt.Sprintf(reGfmCode, "~", "~")),
	itemList:         regexp.MustCompile("(?:[*+-]|\\d+\\.)"),
}

// Inline Grammer
var span = map[itemType]*regexp.Regexp{
	// TODO(Ariel): itemText
	itemItalic: regexp.MustCompile(fmt.Sprintf(reEmphasise, 1)),
	itemStrong: regexp.MustCompile(fmt.Sprintf(reEmphasise, 2)),
	itemStrike: regexp.MustCompile("^~{2}([\\s\\S]+?)~{2}"),
	// itemMixed(e.g: ***str***, ~~*str*~~) will be part of the parser
	itemCode: regexp.MustCompile("^`{1,2}\\s*([\\s\\S]*?[^`])\\s*`{1,2}"),
}
{'fraction_non_alphanumeric': 0.21422903993652473, 'fraction_numerical': 0.01057921184871727, 'mean_word_length': 3.950261780104712, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1383269', 'n_tokens_mistral': 1561, 'n_tokens_neox': 1497, 'n_words': 333}