<commit_before># -*- coding: utf-8 -*-
from postal.text import _normalize
from postal.text.tokenize import tokenize_raw
from postal.text.token_types import token_types

from postal.text.encoding import safe_decode

DEFAULT_STRING_OPTIONS = _normalize.NORMALIZE_STRING_LATIN_ASCII |  \
    _normalize.NORMALIZE_STRING_DECOMPOSE | \
    _normalize.NORMALIZE_STRING_TRIM | \
    _normalize.NORMALIZE_STRING_REPLACE_HYPHENS | \
    _normalize.NORMALIZE_STRING_STRIP_ACCENTS | \
    _normalize.NORMALIZE_STRING_LOWERCASE

DEFAULT_TOKEN_OPTIONS = _normalize.NORMALIZE_TOKEN_REPLACE_HYPHENS | \
    _normalize.NORMALIZE_TOKEN_DELETE_FINAL_PERIOD | \
    _normalize.NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS | \
    _normalize.NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES | \
    _normalize.NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE | \
    _normalize.NORMALIZE_TOKEN_REPLACE_DIGITS


def remove_parens(tokens):
    new_tokens = []
    open_parens = 0
    for t, c in tokens:
        if c == token_types.PUNCT_OPEN:
            open_parens += 1
        elif c == token_types.PUNCT_CLOSE:
            if open_parens > 0:
                open_parens -= 1
        elif open_parens <= 0:
            new_tokens.append((t, c))
    return new_tokens


def normalized_tokens(s, string_options=DEFAULT_STRING_OPTIONS,
                      token_options=DEFAULT_TOKEN_OPTIONS,
                      strip_parentheticals=True):
    '''
    Normalizes a string, tokenizes, and normalizes each token
    with string and token-level options.

    This version only uses libpostal's deterministic normalizations
    i.e. methods with a single output. The string tree version will
    return multiple normalized strings, each with tokens.

    Usage:
        normalized_tokens(u'St.-Barthélemy')
    '''
    s = safe_decode(s)<commit_msg>[python/normalize] importing options from the C module<commit_after># -*- coding: utf-8 -*-
from postal.text import _normalize
from postal.text.tokenize import tokenize_raw
from postal.text.token_types import token_types

from postal.text.encoding import safe_decode

# String options
NORMALIZE_STRING_LATIN_ASCII = _normalize.NORMALIZE_STRING_LATIN_ASCII
NORMALIZE_STRING_TRANSLITERATE = _normalize.NORMALIZE_STRING_TRANSLITERATE
NORMALIZE_STRING_STRIP_ACCENTS = _normalize.NORMALIZE_STRING_STRIP_ACCENTS
NORMALIZE_STRING_DECOMPOSE = _normalize.NORMALIZE_STRING_DECOMPOSE
NORMALIZE_STRING_LOWERCASE = _normalize.NORMALIZE_STRING_LOWERCASE
NORMALIZE_STRING_TRIM = _normalize.NORMALIZE_STRING_TRIM
NORMALIZE_STRING_REPLACE_HYPHENS = _normalize.NORMALIZE_STRING_REPLACE_HYPHENS

DEFAULT_STRING_OPTIONS = NORMALIZE_STRING_LATIN_ASCII |  \
    NORMALIZE_STRING_DECOMPOSE | \
    NORMALIZE_STRING_TRIM | \
    NORMALIZE_STRING_REPLACE_HYPHENS | \
    NORMALIZE_STRING_STRIP_ACCENTS | \
    NORMALIZE_STRING_LOWERCASE

# Token options
NORMALIZE_TOKEN_REPLACE_HYPHENS = _normalize.NORMALIZE_TOKEN_REPLACE_HYPHENS
NORMALIZE_TOKEN_DELETE_HYPHENS = _normalize.NORMALIZE_TOKEN_DELETE_HYPHENS
NORMALIZE_TOKEN_DELETE_FINAL_PERIOD = _normalize.NORMALIZE_TOKEN_DELETE_FINAL_PERIOD
NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS = _normalize.NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS
NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES = _normalize.NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES
NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE = _normalize.NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE
NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC = _normalize.NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC
NORMALIZE_TOKEN_REPLACE_DIGITS = _normalize.NORMALIZE_TOKEN_REPLACE_DIGITS

DEFAULT_TOKEN_OPTIONS = NORMALIZE_TOKEN_REPLACE_HYPHENS | \
    NORMALIZE_TOKEN_DELETE_FINAL_PERIOD | \
    NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS | \
    NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES | \
    NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE | \
    NORMALIZE_TOKEN_REPLACE_DIGITS


def remove_parens(tokens):
    new_tokens = []
    open_parens = 0
    for t, c in tokens:
        if c == token_types.PUNCT_OPEN:
            open_parens += 1
        elif c == token_types.PUNCT_CLOSE:
            if open_parens > 0:
                open_parens -= 1
        elif open_parens <= 0:
            new_tokens.append((t, c))
    return new_tokens


def normalized_tokens(s, string_options=DEFAULT_STRING_OPTIONS,
                      token_options=DEFAULT_TOKEN_OPTIONS,
                      strip_parentheticals=True):
    '''
    Normalizes a string, tokenizes, and normalizes each token
    with string and token-level options.

    This version only uses libpostal's deterministic normalizations
    i.e. methods with a single output. The string tree version will
    return multiple normalized strings, each with tokens.

    Usage:
        normalized_tokens(u'St.-Barthélemy')
    '''
    s = safe_decode(s)
{'fraction_non_alphanumeric': 0.05554382259767687, 'fraction_numerical': 0.0025343189017951427, 'mean_word_length': 4.227373068432671, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5693865', 'n_tokens_mistral': 1683, 'n_tokens_neox': 1647, 'n_words': 259}