<commit_before># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 <NAME>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

from __future__ import unicode_literals
import re
import hashlib
import datetime
import unicode_tex
from unidecode import unidecode
from lxml.html.clean import Cleaner
from lxml.html import fromstring, _transform_result
from lxml import etree
from io import StringIO
from titlecase import titlecase

### General string utilities ###

filter_punctuation_alphanum_regex = re.compile(r'.*\w')
def filter_punctuation(lst):
    """
    :param lst: list of strings
    :returns: all the strings that contain at least one alphanumeric character

    >>> filter_punctuation([u'abc',u'ab.',u'/,',u'a-b',u'#=', u'0'])
    [u'abc', u'ab.', u'a-b', u'0']
    """
    return filter(lambda x: filter_punctuation_alphanum_regex.match(x) is not None,
            lst)

def nocomma(lst):
    """
    Join fields using ',' ensuring that it does not appear in the fields
    This is used to output similarity graphs to be visualized with Gephi.

    :param lst: list of strings
    :returns: these strings joined by commas, ensuring they do not contain
        commas themselves

    >>> nocomma([u'a',u'b',u'cd'])
    u'a,b,cd'
    >>> nocomma([u'a,',u'b'])
    u'a,b'
    >>> nocomma([u'abc',u'',u'\\n',u'def'])
    u'abc, , ,def'
    """
    lst = map(lambda x: str(x).replace(',','').replace('\n',''), lst)
    lst = [x or ' ' for x in lst]
    return ','.join(lst)

def ulower(s):
    """
    Converts to unicode and lowercase.
    :param s: a string
    :return: unicode(s).lower()

    >>> ulower('abSc')
    u'absc'
    >>> ulower(None)
    u'none'
    >>> ulower(89)
    u'89'
    """
    return unicode(s).lower()

def nstrip(s):
    """
    Just like unicode.strip(), but works for None too.

    >>> nstrip(None) is None
    True
    >>> nstrip(u'aa')
    u'aa'
    >>> nstrip(u'  aa \\n')
    u'aa'
    """
    return s.strip() if s else None

def remove_diacritics(s):
    """
    Removes diacritics using the `unidecode` package.

    :param: an str or unicode string
    :returns: if str: the same string. if unicode: the unidecoded string.

    >>> remove_diacritics(u'aéèï')
    'aeei'
    >>> remove_diacritics(u'aéè'.encode('utf-8'))
    'a\\xc3\\xa9\\xc3\\xa8'
    """
    return unidecode(s) if type(s) == unicode else s

def iunaccent(s):
    """
    Removes diacritics and case.

    >>> iunaccent(u'BÉPO forever')
    'bepo forever'
    """
    return remove_diacritics(s).lower()


tokenize_space_re = re.compile(r'\s+')
def tokenize(l):
    """
    A (very very simple) tokenizer.

    >>> tokenize(u'Hello world!')
    [u'Hello', u'world!']
    >>> tokenize(u'99\\tbottles\\nof  beeron \\tThe Wall')
    [u'99', u'bottles', u'of', u'beeron', u'The', u'Wall']
    """
    return tokenize_space_re.split(l)

def maybe_recapitalize_title(title):
    """
    Recapitalize a title if it is mostly uppercase

    >>> maybe_recapitalize_title(u'THIS IS CALLED SCREAMING')
    u'This Is Called Screaming'
    >>> maybe_recapitalize_title(u'This is just a normal title')
    u'This is just a normal title'
    >>> maybe_recapitalize_title(u'THIS IS JUST QUITE Awkward')
    u'THIS IS JUST QUITE Awkward'
    """
    nb_upper = 0
    nb_lower = 0
    for i in range(len(title)):
        if title[i].isupper():
            nb_upper += 1
        elif title[i].islower():
            nb_lower += 1

    if nb_upper > nb_lower:
        title = titlecase(title)
    return title

## HTML sanitizing for the title

overescaped_re = re.compile(r'&amp;#(\d+);')
unicode4_re = re.compile(r'(\\u[0-9A-Z]{4})(?![0-9A-Z])')<commit_msg>papers(utils): add clarification on "mostly uppercase"<commit_after># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 <NAME>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

from __future__ import unicode_literals
import re
import hashlib
import datetime
import unicode_tex
from unidecode import unidecode
from lxml.html.clean import Cleaner
from lxml.html import fromstring, _transform_result
from lxml import etree
from io import StringIO
from titlecase import titlecase

### General string utilities ###

filter_punctuation_alphanum_regex = re.compile(r'.*\w')
def filter_punctuation(lst):
    """
    :param lst: list of strings
    :returns: all the strings that contain at least one alphanumeric character

    >>> filter_punctuation([u'abc',u'ab.',u'/,',u'a-b',u'#=', u'0'])
    [u'abc', u'ab.', u'a-b', u'0']
    """
    return filter(lambda x: filter_punctuation_alphanum_regex.match(x) is not None,
            lst)

def nocomma(lst):
    """
    Join fields using ',' ensuring that it does not appear in the fields
    This is used to output similarity graphs to be visualized with Gephi.

    :param lst: list of strings
    :returns: these strings joined by commas, ensuring they do not contain
        commas themselves

    >>> nocomma([u'a',u'b',u'cd'])
    u'a,b,cd'
    >>> nocomma([u'a,',u'b'])
    u'a,b'
    >>> nocomma([u'abc',u'',u'\\n',u'def'])
    u'abc, , ,def'
    """
    lst = map(lambda x: str(x).replace(',','').replace('\n',''), lst)
    lst = [x or ' ' for x in lst]
    return ','.join(lst)

def ulower(s):
    """
    Converts to unicode and lowercase.
    :param s: a string
    :return: unicode(s).lower()

    >>> ulower('abSc')
    u'absc'
    >>> ulower(None)
    u'none'
    >>> ulower(89)
    u'89'
    """
    return unicode(s).lower()

def nstrip(s):
    """
    Just like unicode.strip(), but works for None too.

    >>> nstrip(None) is None
    True
    >>> nstrip(u'aa')
    u'aa'
    >>> nstrip(u'  aa \\n')
    u'aa'
    """
    return s.strip() if s else None

def remove_diacritics(s):
    """
    Removes diacritics using the `unidecode` package.

    :param: an str or unicode string
    :returns: if str: the same string. if unicode: the unidecoded string.

    >>> remove_diacritics(u'aéèï')
    'aeei'
    >>> remove_diacritics(u'aéè'.encode('utf-8'))
    'a\\xc3\\xa9\\xc3\\xa8'
    """
    return unidecode(s) if type(s) == unicode else s

def iunaccent(s):
    """
    Removes diacritics and case.

    >>> iunaccent(u'BÉPO forever')
    'bepo forever'
    """
    return remove_diacritics(s).lower()


tokenize_space_re = re.compile(r'\s+')
def tokenize(l):
    """
    A (very very simple) tokenizer.

    >>> tokenize(u'Hello world!')
    [u'Hello', u'world!']
    >>> tokenize(u'99\\tbottles\\nof  beeron \\tThe Wall')
    [u'99', u'bottles', u'of', u'beeron', u'The', u'Wall']
    """
    return tokenize_space_re.split(l)

def maybe_recapitalize_title(title):
    """
    Recapitalize a title if it is mostly uppercase
    (number of uppercase letters > number of lowercase letters)

    >>> maybe_recapitalize_title(u'THIS IS CALLED SCREAMING')
    u'This Is Called Screaming'
    >>> maybe_recapitalize_title(u'This is just a normal title')
    u'This is just a normal title'
    >>> maybe_recapitalize_title(u'THIS IS JUST QUITE Awkward')
    u'THIS IS JUST QUITE Awkward'
    """
    nb_upper = 0
    nb_lower = 0
    for i in range(len(title)):
        if title[i].isupper():
            nb_upper += 1
        elif title[i].islower():
            nb_lower += 1

    if nb_upper > nb_lower:
        title = titlecase(title)
    return title

## HTML sanitizing for the title

overescaped_re = re.compile(r'&amp;#(\d+);')
unicode4_re = re.compile(r'(\\u[0-9A-Z]{4})(?![0-9A-Z])')
{'fraction_non_alphanumeric': 0.13645092059188976, 'fraction_numerical': 0.009488309047780414, 'mean_word_length': 3.242453282223287, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 116, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4844036', 'n_tokens_mistral': 3144, 'n_tokens_neox': 3012, 'n_words': 932}