tokenizer.py

# coding=utf-8
# Copyright 2018 The Google AI Language Team Authors, The HuggingFace Inc. team,
# and Marco Polignano.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tokenization classes for Italian AlBERTo models."""
import collections
import logging
import os
import re
import logger

try:
    from ekphrasis.classes.preprocessor import TextPreProcessor
    from ekphrasis.classes.tokenizer import SocialTokenizer
    from ekphrasis.dicts.emoticons import emoticons
except ImportError:
    logger.warning(
        "You need to install ekphrasis to use AlBERToTokenizer"
        "pip install ekphrasis"
    )
    from pip._internal import main as pip
    pip(['install', '--user', 'ekphrasis'])
    from ekphrasis.classes.preprocessor import TextPreProcessor
    from ekphrasis.classes.tokenizer import SocialTokenizer
    from ekphrasis.dicts.emoticons import emoticons

try:
    import numpy as np
except ImportError:
    logger.warning(
        "You need to install numpy to use AlBERToTokenizer"
        "pip install numpy"
    )
    from pip._internal import main as pip
    pip(['install', '--user', 'pandas'])
    import pandas as pd

try:
    from transformers import BertTokenizer, WordpieceTokenizer
    from transformers.tokenization_bert import load_vocab
except ImportError:
    logger.warning(
        "You need to install pytorch-transformers to use AlBERToTokenizer"
        "pip install pytorch-transformers"
    )
    from pip._internal import main as pip
    pip(['install', '--user', 'pytorch-transformers'])
    from transformers import BertTokenizer, WordpieceTokenizer
    from transformers.tokenization_bert import load_vocab

text_processor = TextPreProcessor(
    # terms that will be normalized
    normalize=['url', 'email', 'user', 'percent', 'money', 'phone', 'time', 'date', 'number'],
    # terms that will be annotated
    annotate={"hashtag"},
    fix_html=True,  # fix HTML tokens

    unpack_hashtags=True,  # perform word segmentation on hashtags

    # select a tokenizer. You can use SocialTokenizer, or pass your own
    # the tokenizer, should take as input a string and return a list of tokens
    tokenizer=SocialTokenizer(lowercase=True).tokenize,
    dicts=[emoticons]
)

class AlBERTo_Preprocessing(object):
    def __init__(self, do_lower_case=True, **kwargs):
        self.do_lower_case = do_lower_case

    def preprocess(self, text):
        if self.do_lower_case:
            text = text.lower()
        text = str(" ".join(text_processor.pre_process_doc(text)))
        text = re.sub(r'[^a-zA-ZÀ-ú</>!?♥♡\s\U00010000-\U0010ffff]', ' ', text)
        text = re.sub(r'\s+', ' ', text)
        text = re.sub(r'(\w)\1{2,}', r'\1\1', text)
        text = re.sub(r'^\s', '', text)
        text = re.sub(r'\s$', '', text)
        return text

class AlBERToTokenizer(BertTokenizer):

    def __init__(self, vocab_file, do_lower_case=True,
                 do_basic_tokenize=True, do_char_tokenize=False, do_wordpiece_tokenize=False, do_preprocessing = True, unk_token='[UNK]',
                 sep_token='[SEP]',
                 pad_token='[PAD]', cls_token='[CLS]', mask_token='[MASK]', **kwargs):
        super(BertTokenizer, self).__init__(
            unk_token=unk_token, sep_token=sep_token, pad_token=pad_token,
            cls_token=cls_token, mask_token=mask_token, **kwargs)

        self.do_wordpiece_tokenize = do_wordpiece_tokenize
        self.do_lower_case = do_lower_case
        self.vocab_file = vocab_file
        self.do_basic_tokenize = do_basic_tokenize
        self.do_char_tokenize = do_char_tokenize
        self.unk_token = unk_token
        self.do_preprocessing = do_preprocessing

        if not os.path.isfile(vocab_file):
            raise ValueError(
                "Can't find a vocabulary file at path '{}'.".format(vocab_file))

        self.vocab = load_vocab(vocab_file)
        self.ids_to_tokens = collections.OrderedDict(
            [(ids, tok) for tok, ids in self.vocab.items()])

        if do_wordpiece_tokenize:
            self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab,
                                                          unk_token=self.unk_token)
            
        self.base_bert_tok = BertTokenizer(vocab_file=self.vocab_file, do_lower_case=do_lower_case,
                                      unk_token=unk_token, sep_token=sep_token, pad_token=pad_token,
                                      cls_token=cls_token, mask_token=mask_token, **kwargs)

    def _convert_token_to_id(self, token):
        """Converts a token (str/unicode) to an id using the vocab."""
        # if token[:2] == '##':
        #     token = token[2:]

        return self.vocab.get(token, self.vocab.get(self.unk_token))

    def convert_token_to_id(self, token):
        return self._convert_token_to_id(token)

        return self.vocab.get(token, self.vocab.get(self.unk_token))

    def _convert_id_to_token(self, id):
        # if token[:2] == '##':
        #     token = token[2:]

        return list(self.vocab.keys())[int(id)]
    def convert_id_to_token(self, id):
        return self._convert_id_to_token(id)

    def _convert_tokens_to_string(self,tokens):
        """Converts a sequence of tokens (string) to a single string."""
        out_string = ' '.join(tokens).replace('##', '').strip()
        return out_string

    def convert_tokens_to_string(self,tokens):
        return self._convert_tokens_to_string(tokens)

    def _tokenize(self, text, never_split=None, **kwargs):
        if self.do_preprocessing:
            if self.do_lower_case:
                text = text.lower()
            text = str(" ".join(text_processor.pre_process_doc(text)))
            text = re.sub(r'[^a-zA-ZÀ-ú</>!?♥♡\s\U00010000-\U0010ffff]', ' ', text)
            text = re.sub(r'\s+', ' ', text)
            text = re.sub(r'(\w)\1{2,}', r'\1\1', text)
            text = re.sub(r'^\s', '', text)
            text = re.sub(r'\s$', '', text)
            # print(s)

        split_tokens = [text]
        if self.do_wordpiece_tokenize:
            wordpiece_tokenizer = WordpieceTokenizer(self.vocab,self.unk_token)
            split_tokens = wordpiece_tokenizer.tokenize(text)

        elif self.do_char_tokenize:
            tokenizer = CharacterTokenizer(self.vocab, self.unk_token)
            split_tokens = tokenizer.tokenize(text)

        elif self.do_basic_tokenize:
            """Tokenizes a piece of text."""
            split_tokens = self.base_bert_tok.tokenize(text)

        return split_tokens

    def tokenize(self, text, never_split=None, **kwargs):
        return self._tokenize(text, never_split)


class CharacterTokenizer(object):
    """Runs Character tokenziation."""

    def __init__(self, vocab, unk_token,
                 max_input_chars_per_word=100, with_markers=True):
        """Constructs a CharacterTokenizer.
        Args:
            vocab: Vocabulary object.
            unk_token: A special symbol for out-of-vocabulary token.
            with_markers: If True, "#" is appended to each output character except the
                first one.
        """
        self.vocab = vocab
        self.unk_token = unk_token
        self.max_input_chars_per_word = max_input_chars_per_word
        self.with_markers = with_markers

    def tokenize(self, text):
        """Tokenizes a piece of text into characters.

        For example:
            input = "apple"
            output = ["a", "##p", "##p", "##l", "##e"]  (if self.with_markers is True)
            output = ["a", "p", "p", "l", "e"]          (if self.with_markers is False)
        Args:
            text: A single token or whitespace separated tokens.
                This should have already been passed through `BasicTokenizer`.
        Returns:
            A list of characters.
        """

        output_tokens = []
        for i, char in enumerate(text):
            if char not in self.vocab:
                output_tokens.append(self.unk_token)
                continue

            if self.with_markers and i != 0:
                output_tokens.append('##' + char)
            else:
                output_tokens.append(char)

        return output_tokens

if __name__== "__main__":
    a = AlBERTo_Preprocessing(do_lower_case=True)
    s = "#IlGOverno presenta le linee guida sulla scuola #labuonascuola - http://t.co/SYS1T9QmQN"
    b = a.preprocess(s)
    print(b)

    c =AlBERToTokenizer(do_lower_case=True,vocab_file="vocab.txt", do_preprocessing=True)
    d = c.tokenize(s)
    print(d)