<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

package tokenizer

import (
	"bufio"
	"io"
	"regexp"

	"github.com/ta2gch/iris/runtime/env"
	"github.com/ta2gch/iris/runtime/ilos"
	"github.com/ta2gch/iris/runtime/ilos/class"
	"github.com/ta2gch/iris/runtime/ilos/instance"
)

// Tokenizer interface type is the interface for reading string with every token
type Tokenizer struct {
	sc *bufio.Scanner
}

var re *regexp.Regexp

func Tokenize(r io.Reader) *Tokenizer {
	str := ``
	str += `1\+|1-|`
	str += `[-+]?[[:digit:]]+\.[[:digit:]]+|`
	str += `[-+]?[[:digit:]]+(?:\.[[:digit:]]+)?[eE][-+]?[[:digit:]]+|`
	str += `[-+]?[[:digit:]]+|`
	str += `#[bB][-+]?[01]+|`
	str += `#[oO][-+]?[0-7]+|`
	str += `#[xX][-+]?[[:xdigit:]]+|`
	str += `#\\[[:alpha:]]+|`
	str += `#\\[[:graph:]]|`
	str += `"(?:\\\\|\\"|[^"])*"|`
	str += `[:&][a-zA-Z]+|`
	str += `\+|-|[a-zA-Z<>/*=?_!$%[\]^{}~][-a-zA-Z0-9+<>/*=?_!$%[\]^{}~]*|`
	str += `\|(?:\\\\|\\\||[^|])*\||`
	str += `[.()]|`
	str += ";.*?\n|"
	str += `#\|.*?\|#|`
	str += "#'|,@?|'|`|#[[:digit:]]*[aA]|#" // TODO: hangs at #ab or #3
	re = regexp.MustCompile(str)
	sc := bufio.NewScanner(r)
	sc.Split(splitter)
	return &Tokenizer{sc}
}

func (t *Tokenizer) Next() (string, ilos.Instance) {
	if t.sc.Scan() {
		return t.sc.Text(), nil
	}
	return "", instance.Create(env.NewEnvironment(nil, nil, nil, nil), class.EndOfStream)
}<commit_msg>ignore | in a piped symbol<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

package tokenizer

import (
	"bufio"
	"io"
	"regexp"

	"github.com/ta2gch/iris/runtime/env"
	"github.com/ta2gch/iris/runtime/ilos"
	"github.com/ta2gch/iris/runtime/ilos/class"
	"github.com/ta2gch/iris/runtime/ilos/instance"
)

// Tokenizer interface type is the interface for reading string with every token
type Tokenizer struct {
	sc *bufio.Scanner
}

var re *regexp.Regexp

func Tokenize(r io.Reader) *Tokenizer {
	str := ``
	str += `1\+|1-|`
	str += `[-+]?[[:digit:]]+\.[[:digit:]]+|`
	str += `[-+]?[[:digit:]]+(?:\.[[:digit:]]+)?[eE][-+]?[[:digit:]]+|`
	str += `[-+]?[[:digit:]]+|`
	str += `#[bB][-+]?[01]+|`
	str += `#[oO][-+]?[0-7]+|`
	str += `#[xX][-+]?[[:xdigit:]]+|`
	str += `#\\[[:alpha:]]+|`
	str += `#\\[[:graph:]]|`
	str += `"(?:\\\\|\\"|[^"])*"|`
	str += `[:&][a-zA-Z]+|`
	str += `\+|-|[a-zA-Z<>/*=?_!$%[\]^{}~][-a-zA-Z0-9+<>/*=?_!$%[\]^{}~]*|`
	str += `\|[^|]*\||`
	str += `[.()]|`
	str += ";.*?\n|"
	str += `#\|.*?\|#|`
	str += "#'|,@?|'|`|#[[:digit:]]*[aA]|#" // TODO: hangs at #ab or #3
	re = regexp.MustCompile(str)
	sc := bufio.NewScanner(r)
	sc.Split(splitter)
	return &Tokenizer{sc}
}

func (t *Tokenizer) Next() (string, ilos.Instance) {
	if t.sc.Scan() {
		return t.sc.Text(), nil
	}
	return "", instance.Create(env.NewEnvironment(nil, nil, nil, nil), class.EndOfStream)
}
{'fraction_non_alphanumeric': 0.3177539759818241, 'fraction_numerical': 0.01103537812398572, 'mean_word_length': 3.8081123244929795, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5691510', 'n_tokens_mistral': 1476, 'n_tokens_neox': 1412, 'n_words': 264}