<commit_before># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 <NAME>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

from __future__ import unicode_literals

import re
import name_tools

from papers.utils import split_words, iunaccent, remove_diacritics, isupper

# Name managemement: heuristics to separate a name into (first,last)
comma_re = re.compile(r',+')
space_re = re.compile(r'\s+')
initial_re = re.compile(r'(^|\W)\w(\W|$)')
lowercase_re = re.compile(r'[a-z]')
letter_re = re.compile(r'\w')

def match_names(a,b):
    """
    Returns a boolean: are these two names compatible?
    Examples:
    > ('Robin', 'Ryder'),('R.', 'Ryder'): True
    > ('<NAME>.', 'Ryder'),('R.', 'Ryder'): True
    > ('<NAME>.', 'Ryder'),('J.', 'Ryder'): False
    > ('Claire', 'Mathieu'),('Claire', 'Kenyon-Mathieu'): False
    """
    if not a or not b:
        return False
    (firstA,lastA) = a
    (firstB,lastB) = b
    if lastA.lower() != lastB.lower():
        return False
    partsA = split_words(firstA)
    partsB = split_words(firstB)
    parts = zip(partsA, partsB)
    return all(map(match_first_names, parts))

initial_re = re.compile(r'[A-Z](\.,;)*$')
def normalize_name_words(w):
    """ If it is an initial, ensure it is of the form "T.", and recapitalize fully capitalized words. """
    w = w.strip()
    words = w.split()
    words = map(recapitalize_word, words)
    words = map(lambda w: w[0]+'.' if initial_re.match(w) else w, words)
    return ' '.join(words)


def recapitalize_word(w):
    """ Turns every fully capitalized word into an uncapitalized word (except for the first character) """
    if w.upper() == w:
        previousIsChar = False
        res = ''
        for i in range(len(w)):
            if previousIsChar:
                res += w[i].lower()
            else:
                res += w[i]
            previousIsChar = letter_re.match(w[i]) != None
        return res
    return w

def match_first_names(pair):
    a,b = pair
    if len(a) == 1 and len(b) > 0:
        return a.lower() == b[0].lower()
    elif len(b) == 1 and len(a) > 0:
        return b.lower() == a[0].lower()
    else:
        return a.lower() == b.lower()

def to_plain_name(name):
    return (name.first,name.last)

# Name normalization function used by the OAI proxy
nn_separator_re = re.compile(r',+ *')
nn_escaping_chars_re = re.compile(r'[\{\}\\]')
nn_nontext_re = re.compile(r'[^a-z_]+')
nn_final_nontext_re = re.compile(r'[^a-z_]+$')

def name_normalization(ident):
    ident = remove_diacritics(ident).lower()
    ident = ident.strip()
    ident = nn_separator_re.sub('_',ident)
    ident = nn_escaping_chars_re.sub('',ident)
    ident = nn_final_nontext_re.sub('',ident)
    ident = nn_nontext_re.sub('-',ident)
    return ident

def name_signature(first, last):
    ident = last.lower().strip()
    ident = nn_escaping_chars_re.sub('',ident)
    ident = nn_final_nontext_re.sub('',ident)
    ident = nn_nontext_re.sub('-',ident)
    if len(first):
        ident = first[0].lower()+'-'+ident
    return ident

#### Helpers for the name splitting heuristic ######

# Does this string contain a name initial?
def contains_initials(s):
    return initial_re.search(iunaccent(s)) != None
# Is this word fully capitalized?
def is_fully_capitalized(s):
    return lowercase_re.search(remove_diacritics(s)) == None
# Split a word according to a predicate
def predsplit_forward(predicate, words):
    first = []
    last = []
    predHolds = True
    for i in range(len(words)):
        if predicate(i) and predHolds:
            first.append(words[i])
        else:
            predHolds = False
            last.append(words[i])
    return (first,last)
# The same, but backwards
def predsplit_backwards(predicate, words):
    first = []
    last = []
    predHolds = True
    for i in reversed(range(len(words))):
        if predicate(i) and predHolds:
            last.insert(0, words[i])
        else:
            predHolds = False<commit_msg>Draft of a name similarity score<commit_after># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 <NAME>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

from __future__ import unicode_literals

import re
import name_tools

from papers.utils import split_words, iunaccent, remove_diacritics, isupper

# Name managemement: heuristics to separate a name into (first,last)
comma_re = re.compile(r',+')
space_re = re.compile(r'\s+')
initial_re = re.compile(r'(^|\W)\w(\W|$)')
lowercase_re = re.compile(r'[a-z]')
letter_re = re.compile(r'\w')

def match_names(a,b):
    """
    Returns a boolean: are these two names compatible?
    Examples:
    > ('Robin', 'Ryder'),('R.', 'Ryder'): True
    > ('<NAME>.', 'Ryder'),('R.', 'Ryder'): True
    > ('<NAME>.', 'Ryder'),('J.', 'Ryder'): False
    > ('Claire', 'Mathieu'),('Claire', 'Kenyon-Mathieu'): False
    """
    return name_similarity(a,b) > 0.

initial_re = re.compile(r'[A-Z](\.,;)*$')
def normalize_name_words(w):
    """ If it is an initial, ensure it is of the form "T.", and recapitalize fully capitalized words. """
    w = w.strip()
    words = w.split()
    words = map(recapitalize_word, words)
    words = map(lambda w: w[0]+'.' if initial_re.match(w) else w, words)
    return ' '.join(words)


def recapitalize_word(w):
    """ Turns every fully capitalized word into an uncapitalized word (except for the first character) """
    if w.upper() == w:
        previousIsChar = False
        res = ''
        for i in range(len(w)):
            if previousIsChar:
                res += w[i].lower()
            else:
                res += w[i]
            previousIsChar = letter_re.match(w[i]) != None
        return res
    return w

def match_first_names(pair):
    a,b = pair
    if len(a) == 1 and len(b) > 0:
        return a.lower() == b[0].lower()
    elif len(b) == 1 and len(a) > 0:
        return b.lower() == a[0].lower()
    else:
        return a.lower() == b.lower()

def to_plain_name(name):
    return (name.first,name.last)

# Name normalization function used by the OAI proxy
nn_separator_re = re.compile(r',+ *')
nn_escaping_chars_re = re.compile(r'[\{\}\\]')
nn_nontext_re = re.compile(r'[^a-z_]+')
nn_final_nontext_re = re.compile(r'[^a-z_]+$')

def name_normalization(ident):
    ident = remove_diacritics(ident).lower()
    ident = ident.strip()
    ident = nn_separator_re.sub('_',ident)
    ident = nn_escaping_chars_re.sub('',ident)
    ident = nn_final_nontext_re.sub('',ident)
    ident = nn_nontext_re.sub('-',ident)
    return ident

def name_signature(first, last):
    ident = last.lower().strip()
    ident = nn_escaping_chars_re.sub('',ident)
    ident = nn_final_nontext_re.sub('',ident)
    ident = nn_nontext_re.sub('-',ident)
    if len(first):
        ident = first[0].lower()+'-'+ident
    return ident

### Name similarity measure ###

def name_similarity(a,b):
    """
    Returns a float: how similar are these two names?
    Examples:
    > ('Robin', 'Ryder'),('Robin', 'Ryder'): 1.0
    > ('Robin', 'Ryder'),('R.', 'Ryder'): 0.5
    > ('R.', 'Ryder'),('R.', 'Ryder'): 0.5
    > ('<NAME>.', 'Ryder'),('R.', 'Ryder'): 0.5
    > ('<NAME>.', 'Ryder'),('J.', 'Ryder'): 0
    > ('Claire', 'Mathieu'),('Claire', 'Kenyon-Mathieu'): 0
    > ('Robin', 'Ryder'), ('<NAME>.', 'Ryder') : 1
    > ('<NAME>.','Ryder'), ('<NAME>.', 'Ryder'): 0.
    """
    if not a or not b:
        return False
    (firstA,lastA) = a
    (firstB,lastB) = b
    if lastA.lower() != lastB.lower():
        return False
    partsA = split_words(firstA)
    partsB = split_words(firstB)
    parts = zip(partsA, partsB)
    if not all(map(match_first_names, parts)):
        return 0.
    full_name_found = (True in map(lambda (a,b): len(a) > 1 and len(b) > 1, parts))
    if full_name_found:
        return 1.
    return 0.5


#### Helpers for the name splitting heuristic ######

# Does this string contain a name initial?
def contains_initials(s):
    return initial_re.search(iunaccent(s)) != None
# Is this word fully capitalized?
def is_fully_capitalized(s):
    return lowercase_re.search(remove_diacritics(s)) == None
# Split a word according to a predicate
def predsplit_forward(predicate, words):
    first = []
    last = []
    predHolds = True
    for i in range(len(words)):
        if predicate(i) and predHolds:
            first.append(words[i])
        else:
            predHolds = False
            last.append(words[i])
    return (first,last)
# The same, but backwards
def predsplit_backwards(predicate, words):
    first = []
    last = []
    predHolds = True
    for i in reversed(range(len(words))):
        if predicate(i) and predHolds:
            last.insert(0, words[i])
        else:
            predHolds = False
{'fraction_non_alphanumeric': 0.14774078478002378, 'fraction_numerical': 0.007035275465715418, 'mean_word_length': 3.1689384551838082, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2795219', 'n_tokens_mistral': 3473, 'n_tokens_neox': 3355, 'n_words': 981}