<commit_before>package gotiscrape

import "regexp"

var md5 = regexp.MustCompile("\\b[0-9a-zA-Z]{32}\\b")
var sha1 = regexp.MustCompile("\\b[.0-9a-zA-Z]{40}\\b")
var sha2 = regexp.MustCompile("\\b[0-9a-zA-Z]{64}\\b")
var ipv4 = regexp.MustCompile("\\b([0-2]?[0-9]?[0-9])\\.([0-2]?[0-9]?[0-9])" +
	"\\.([0-2]?[0-9]?[0-9])\\.([0-2]?[0-9]?[0-9])\\b")
var email = regexp.MustCompile("\\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+" +
	"\\.[a-zA-Z]{2,4}\\b")
var fqdn = regexp.MustCompile("\\b([a-zA-Z0-9\\.\\-]+" +
	"\\.[a-zA-Z0-9\\.\\-]+)\\b")

var allposipv6 = regexp.MustCompile("\\b[a-f0-9:.]+\\b")<commit_msg>fix in hexdigest patterns.. A-F not A-Z<commit_after>package gotiscrape

import "regexp"

var md5 = regexp.MustCompile("\\b[0-9a-fA-F]{32}\\b")
var sha1 = regexp.MustCompile("\\b[.0-9a-fA-F]{40}\\b")
var sha2 = regexp.MustCompile("\\b[0-9a-fA-F]{64}\\b")
var ipv4 = regexp.MustCompile("\\b([0-2]?[0-9]?[0-9])\\.([0-2]?[0-9]?[0-9])" +
	"\\.([0-2]?[0-9]?[0-9])\\.([0-2]?[0-9]?[0-9])\\b")
var email = regexp.MustCompile("\\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+" +
	"\\.[a-zA-Z]{2,4}\\b")
var fqdn = regexp.MustCompile("\\b([a-zA-Z0-9\\.\\-]+" +
	"\\.[a-zA-Z0-9\\.\\-]+)\\b")

var allposipv6 = regexp.MustCompile("\\b[a-f0-9:.]+\\b")
{'fraction_non_alphanumeric': 0.38235294117647056, 'fraction_numerical': 0.08660130718954248, 'mean_word_length': 2.990228013029316, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5949932', 'n_tokens_mistral': 720, 'n_tokens_neox': 685, 'n_words': 61}