<commit_before>package token

//Comma is true if v is a literal comma.
func (v Value) Comma() bool {
	return v.Kind == Literal && v.Value == ","
}

//Op is true if v is literal and first rune is one of "!@%&|-+=/<>"".
func (v Value) Op() bool {
	if v.Kind != Literal {
		return false
	}
	s := v.Value[0]
	for _, r := range "!@%&|-+=/<>*" {
		if byte(r) == s {
			return true
		}
	}
	return false
}<commit_msg>fixed error in sql unparser preventing dot from being handled correctly<commit_after>package token

//Comma is true if v is a literal comma.
func (v Value) Comma() bool {
	return v.Kind == Literal && v.Value == ","
}

//Op is true if v is literal and first rune is one of "!@%&|-+=/<>"".
func (v Value) Op() bool {
	if v.Kind != Literal {
		return false
	}
	s := v.Value[0]
	for _, r := range "!@%&|-+=/<>*." {
		if byte(r) == s {
			return true
		}
	}
	return false
}
{'fraction_non_alphanumeric': 0.18586088939566706, 'fraction_numerical': 0.002280501710376283, 'mean_word_length': 2.8849557522123894, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3354819', 'n_tokens_mistral': 351, 'n_tokens_neox': 333, 'n_words': 120}