<commit_before>import unicodedata
import string
import re

def generalize_names(name, output_sep=' ', firstname_output_letters=1):
    """
    Function that outputs a person's name in the format 
    <last_name><separator><firstname letter(s)> (all lowercase)
        
    Parameters
    ----------
    name : `str`
      Name of the player
    
    output_sep : `str` (default: ' ')
      String for separating last name and first name in the output.
    
    firstname_output_letters : `int`
      Number of letters in the abbreviated first name.
      
    Returns
    ----------
    gen_name : `str`
      The generalized name.
        
    """
    # set first and last name positions
    last, first = 'last', 'first'
    last_pos = -1
    
    name = name.lower()
    
    # fix primarily Dutch names
    exc = ['<NAME> ', 'de ', 'van ', 'von ']
    for e in exc:
        if name.startswith(e):
            repl = e.replace(' ','')
            name = (repl + name[len(e)-1:].strip())
            
    exc = [' <NAME> ', ' de', ' van ', ' von ', ', <NAME> ', ', de', ', van ', ', von ']
    for e in exc:
        name = name.replace(e, ' '+e.replace(' ', ''))
            
        <commit_msg>final update for generalize names<commit_after>import unicodedata
import string
import re

def generalize_names(name, output_sep=' ', firstname_output_letters=1):
    """
    Function that outputs a person's name in the format 
    <last_name><separator><firstname letter(s)> (all lowercase)
        
    Parameters
    ----------
    name : `str`
      Name of the player
    
    output_sep : `str` (default: ' ')
      String for separating last name and first name in the output.
    
    firstname_output_letters : `int`
      Number of letters in the abbreviated first name.
      
    Returns
    ----------
    gen_name : `str`
      The generalized name.
        
    """
    # set first and last name positions
    last, first = 'last', 'first'
    last_pos = -1
    
    name = name.lower()
    
    # fix primarily Dutch names
    exc = ['<NAME> ', 'de ', 'van ', 'von ', 'di ']
    for e in exc:
        if name.startswith(e):
            repl = e.replace(' ','')
            name = (repl + name[len(e)-1:].strip())
            
    exc = [' <NAME> ', ' de ', ' van ', ' von ', ' di ',
    ', <NAME> ', ', de', ', van ', ', von ', ', di ']
    
    for e in exc:
        name = name.replace(e, ' '+e.replace(' ', ''))
            
        
{'fraction_non_alphanumeric': 0.1493877551020408, 'fraction_numerical': 0.0024489795918367346, 'mean_word_length': 1.8801410105757932, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6305221', 'n_tokens_mistral': 767, 'n_tokens_neox': 741, 'n_words': 227}