<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    This module contains functions and constants used to clean and filter
    words in texts.
    Some third party modules can be used with this module, if you want to improve it
    * python-nltk for more stopwords;
    * rfc3987 to better identify URLS;
    * a lot of twitter third party API's to identify hashtags, URLS etc.

    The libraries above are not currently used to make the script easier to install 
    for end users.
"""
import string
from customized_stopwords import portuguese_common_words

""" 
This list is defined in it's own file in order to make modifications 
easier for non-programmers.
Check python ntlk for better stopwords in your language.
"""
CUSTOMIZED_STOPWORDS = set(portuguese_common_words)

VALID_CHARACTERS = string.ascii_letters + string.digits
EXTRA_CHARACTERS = "_-"
VALID_CHARACTERS = VALID_CHARACTERS + EXTRA_CHARACTERS

VALID_CHARACTERS_SET = set([])

for character in VALID_CHARACTERS:
    VALID_CHARACTERS_SET.add(character)

ACCENT_REPLACEMENTS = {
    ord('á'):'a',
    ord('à'):'a',
    ord('è'):'e',
    ord('ê'):'e',
    ord('é'):'e',
    ord('í'):'i',
    ord('ì'):'i',
    ord('ò'):'o',
    ord('ó'):'o',
    ord('ô'):'o',
    ord('õ'):'o',
    ord('ù'):'u',
    ord('ú'):'u',
    ord('ü'):'u',
    ord('ç'):'c'
}

"""
Characters to be excluded from the strings. Some characters not covered<commit_msg>Added the ã and â accent replacement.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    This module contains functions and constants used to clean and filter
    words in texts.
    Some third party modules can be used with this module, if you want to improve it
    * python-nltk for more stopwords;
    * rfc3987 to better identify URLS;
    * a lot of twitter third party API's to identify hashtags, URLS etc.

    The libraries above are not currently used to make the script easier to install 
    for end users.
"""
import string
from customized_stopwords import portuguese_common_words, english_common_words, spanish_common_words

""" 
This list is defined in it's own file in order to make modifications 
easier for non-programmers.
Check python ntlk for better stopwords in your language.
"""
CUSTOMIZED_STOPWORDS = set(portuguese_common_words+ spanish_common_words + english_common_words)

VALID_CHARACTERS = string.ascii_letters + string.digits
EXTRA_CHARACTERS = "_-"
VALID_CHARACTERS = VALID_CHARACTERS + EXTRA_CHARACTERS

VALID_CHARACTERS_SET = set([])

for character in VALID_CHARACTERS:
    VALID_CHARACTERS_SET.add(character)

ACCENT_REPLACEMENTS = {
    ord('á'):'a',
    ord('ã'):'a',
    ord('â'):'a',
    ord('à'):'a',
    ord('è'):'e',
    ord('ê'):'e',
    ord('é'):'e',
    ord('í'):'i',
    ord('ì'):'i',
    ord('ò'):'o',
    ord('ó'):'o',
    ord('ô'):'o',
    ord('õ'):'o',
    ord('ù'):'u',
    ord('ú'):'u',
    ord('ü'):'u',
    ord('ç'):'c'
}

"""
Characters to be excluded from the strings. Some characters not covered
{'fraction_non_alphanumeric': 0.1362571141613659, 'fraction_numerical': 0.0033478406427854034, 'mean_word_length': 3.9064039408866993, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1395723', 'n_tokens_mistral': 1026, 'n_tokens_neox': 995, 'n_words': 303}