<commit_before>package scanner

import "strings"

var (
	symbols = map[string]struct{}{
		".":  struct{}{},
		"->": struct{}{},
		"{":  struct{}{},
		"}":  struct{}{},
		"[":  struct{}{},
		"]":  struct{}{},
		"=>": struct{}{},
		";":  struct{}{},
	}

	keywords = map[string]struct{}{
		"switch":  struct{}{},
		"default": struct{}{},
	}
)

func isKeyword(str string) bool {
	_, ok := keywords[str]
	return ok
}

func symbolicSuffix(str string) string {
	for k := range symbols {
		if strings.HasSuffix(str, k) {
			return k
		}
	}

	return ""
}

func isQuote(r rune) bool {
	return (r == '\'') || (r == '"')
}
<commit_msg>scanner: Handle parantheses.<commit_after>package scanner

import "strings"

var (
	symbols = map[string]struct{}{
		".":  struct{}{},
		"->": struct{}{},
		"{":  struct{}{},
		"}":  struct{}{},
		"[":  struct{}{},
		"]":  struct{}{},
		"(":  struct{}{},
		")":  struct{}{},
		"=>": struct{}{},
		";":  struct{}{},
	}

	keywords = map[string]struct{}{
		"switch":  struct{}{},
		"default": struct{}{},
	}
)

func isKeyword(str string) bool {
	_, ok := keywords[str]
	return ok
}

func symbolicSuffix(str string) string {
	for k := range symbols {
		if strings.HasSuffix(str, k) {
			return k
		}
	}

	return ""
}

func isQuote(r rune) bool {
	return (r == '\'') || (r == '"')
}

{'fraction_non_alphanumeric': 0.25672559569561876, 'fraction_numerical': 0.0, 'mean_word_length': 3.489655172413793, 'pattern_counts': {'":': 22, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5474550', 'n_tokens_mistral': 545, 'n_tokens_neox': 498, 'n_words': 104}