def _word_tokenizer(text: str) -> List[str]:
    tokens: List[str] = []
    for paragraph in segmenter.analyze(text):
        for sentence in paragraph:
            for token in sentence:
                token_value = token.value
                for c in ('/', '\\*', "'", '\\.', '--', ':'):
                    token_value = re.sub(f'({c})', ' \\1 ', token_value)
                tokens.extend(token_value.split())
    return tokens
def _word_tokenizer(text: str) -> List[str]:
    tokens: List[str] = []
    for paragraph in segmenter.analyze(text):
        for sentence in paragraph:
            for token in sentence:
                token_value = token.value
                for c in ('/', '\\*', "'", '\\.', '--', ':'):
                    token_value = re.sub(f'({c})', ' \\1 ', token_value)
                tokens.extend(token_value.split())
    return tokens