<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import wikipedia, re, string
import math, sys

MAX_USERNAME_LENGTH = 31

RE_LINKS = re.compile('(\[\[[Uu]ser *: *(.{1,%i}?) *(?:\| *(.+?) *)?\]\])' % MAX_USERNAME_LENGTH)

# anything within a [[User:x|y]] style link 
re_userlink = '\[\[[Uu]ser\s*:\s*(.{1,%i}?)\s*(?:\|\s*(?:.+?)\s*)?\]\]' % MAX_USERNAME_LENGTH
re_userstring = '([\w^,&\']{1,%i})' % MAX_USERNAME_LENGTH
# the same, or just an arbitrary string
re_maybelink = '(?:'+re_userlink+'|'+re_userstring+')'
# any enumerator
re_enumerator = "(?:,| and | with |&)"

# a sequence of re_maybelinks, separated by re_enumerators
re_maybelist = re_maybelink+'(?:'+re_enumerator+re_maybelink+')*'
# a sequence of re_userlinks, separated by re_enumerators
re_strictlylist = '(?:'+re_userlink+'(?:'+re_enumerator+re_userlink+')+)'
# either a re_userlink, or a re_strictlylist
re_linkorlist = '(?:'+re_strictlylist+'|'+re_userlink+')'

# an option to a wiki template
re_option   = '\s*([^=]+?)(?:\}|\|\s*\w+\s*=)'

RE_USERLINK = re.compile(re_userlink)
RE_LISTED = re.compile('\s*[\*]\s*('+re_maybelist+')[^\n]*')
RE_LISTEDLINK = re.compile('\s*[\*].*?('+re_linkorlist+')[^\n]*')
RE_RIBBONBEARER = re.compile('\{\{.*?\|\s*name\s*='+re_option, re.DOTALL)
RE_CARDRECIPIENT = re.compile('recipient ?='+re_option)
RE_ENTITLED = re.compile('==+\s*('+re_linkorlist+')\s*=+=')
RE_MEETUP = re.compile('\{\{\s*[Mm]eet-up.*?\|\s*name\s*='+re_option, re.DOTALL)
RE_FIRST = re.compile('^.*?'+re_userlink, re.DOTALL)
RE_COMMONPLACES = re.compile('(?:reached by)\s+('+re_maybelist+')\s*\.')
RE_BOLDED = re.compile('\\\'{3}('+re_maybelist+')') #does not work!
RE_PARALIST = re.compile('\n\n('+re_maybelink+').*?(?=\n\n)', re.MULTILINE ^ re.DOTALL)

improbablenames = ["", " ", "a", "and", "i", "i'll", "we", "the", "one", "two", "three", "all of us", "all attendees", "everyone", "his", "her", "probably", "drag", "drag-along", "1", "2", "3", "4", "5"]

debug_fuzz = None
debug_links = None
def getDebugFuzz():
  global debug_fuzz
  return debug_fuzz

def getDebugLinks():
  global debug_links
  return debug_links

def normalize(dic):
  maxfuzz = 0<commit_msg>just a few more improbable names<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import wikipedia, re, string
import math, sys

MAX_USERNAME_LENGTH = 31

RE_LINKS = re.compile('(\[\[[Uu]ser *: *(.{1,%i}?) *(?:\| *(.+?) *)?\]\])' % MAX_USERNAME_LENGTH)

# anything within a [[User:x|y]] style link 
re_userlink = '\[\[[Uu]ser\s*:\s*(.{1,%i}?)\s*(?:\|\s*(?:.+?)\s*)?\]\]' % MAX_USERNAME_LENGTH
re_userstring = '([\w^,&\']{1,%i})' % MAX_USERNAME_LENGTH
# the same, or just an arbitrary string
re_maybelink = '(?:'+re_userlink+'|'+re_userstring+')'
# any enumerator
re_enumerator = "(?:,| and | with |&)"

# a sequence of re_maybelinks, separated by re_enumerators
re_maybelist = re_maybelink+'(?:'+re_enumerator+re_maybelink+')*'
# a sequence of re_userlinks, separated by re_enumerators
re_strictlylist = '(?:'+re_userlink+'(?:'+re_enumerator+re_userlink+')+)'
# either a re_userlink, or a re_strictlylist
re_linkorlist = '(?:'+re_strictlylist+'|'+re_userlink+')'

# an option to a wiki template
re_option   = '\s*([^=]+?)(?:\}|\|\s*\w+\s*=)'

RE_USERLINK = re.compile(re_userlink)
RE_LISTED = re.compile('\s*[\*]\s*('+re_maybelist+')[^\n]*')
RE_LISTEDLINK = re.compile('\s*[\*].*?('+re_linkorlist+')[^\n]*')
RE_RIBBONBEARER = re.compile('\{\{.*?\|\s*name\s*='+re_option, re.DOTALL)
RE_CARDRECIPIENT = re.compile('recipient ?='+re_option)
RE_ENTITLED = re.compile('==+\s*('+re_linkorlist+')\s*=+=')
RE_MEETUP = re.compile('\{\{\s*[Mm]eet-up.*?\|\s*name\s*='+re_option, re.DOTALL)
RE_FIRST = re.compile('^.*?'+re_userlink, re.DOTALL)
RE_COMMONPLACES = re.compile('(?:reached by)\s+('+re_maybelist+')\s*\.')
RE_BOLDED = re.compile('\\\'{3}('+re_maybelist+')') #does not work!
RE_PARALIST = re.compile('\n\n('+re_maybelink+').*?(?=\n\n)', re.MULTILINE ^ re.DOTALL)

improbablenames = ["", " ", "a", "and", "i", "i'll", "we", "the", "one", "two", "three", "all of us", "all attendees", "everyone", "his", "her", "probably", "drag", "drag-along", "1", "2", "3", "4", "5", "wife", "family", "friends"]

debug_fuzz = None
debug_links = None
def getDebugFuzz():
  global debug_fuzz
  return debug_fuzz

def getDebugLinks():
  global debug_links
  return debug_links

def normalize(dic):
  maxfuzz = 0
{'fraction_non_alphanumeric': 0.26056014692378326, 'fraction_numerical': 0.005968778696051423, 'mean_word_length': 5.776049766718507, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '310832', 'n_tokens_mistral': 1901, 'n_tokens_neox': 1850, 'n_words': 317}