codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for mgtstr char import json import os from typing import optional tuple from tokenizationutils import pretrainedtokenizer from utils import logging logger logging getloggername vocabfilesnames vocabfile vocab json pretrainedvocabfilesmap vocabfile mgpstr https huggingface coalibabadamomgpstrbaseblobmainvocab json pretrainedpositionalembeddingssizes mgpstr 27 class mgpstrtokenizerpretrainedtokenizer vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes def initself vocabfile unktokengo bostokengo eostokens padtokengo kwargs with openvocabfile encodingutf8 as vocabhandle self vocab json loadvocabhandle self decoder v k for k v in self vocab items super init unktokenunktoken bostokenbostoken eostokeneostoken padtokenpadtoken kwargs property def vocabsizeself return lenself vocab def getvocabself vocab dictself vocab copy vocab updateself addedtokensencoder return vocab def tokenizeself text converts a token str in an id using the vocab return self vocab gettoken self vocab getself unktoken def convertidtotokenself index coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for mgt str char construct a mgp str char tokenizer this tokenizer inherits from pretrainedtokenizer which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str path to the vocabulary file unk_token str optional defaults to go the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead bos_token str optional defaults to go the beginning of sequence token eos_token str optional defaults to s the end of sequence token pad_token str or tokenizers addedtoken optional defaults to go a special token used to make arrays of tokens the same size for batching purpose will then be ignored by attention mechanisms or loss computation tokenize a string converts a token str in an id using the vocab converts an index integer in a token str using the vocab
import json
import os
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "vocab.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "mgp-str": "https://huggingface.co/alibaba-damo/mgp-str-base/blob/main/vocab.json",
    }
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"mgp-str": 27}


class MgpstrTokenizer(PreTrainedTokenizer):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

    def __init__(self, vocab_file, unk_token="[GO]", bos_token="[GO]", eos_token="[s]", pad_token="[GO]", **kwargs):
        with open(vocab_file, encoding="utf-8") as vocab_handle:
            self.vocab = json.load(vocab_handle)
        self.decoder = {v: k for k, v in self.vocab.items()}
        super().__init__(
            unk_token=unk_token,
            bos_token=bos_token,
            eos_token=eos_token,
            pad_token=pad_token,
            **kwargs,
        )

    @property
    def vocab_size(self):
        return len(self.vocab)

    def get_vocab(self):
        vocab = dict(self.vocab).copy()
        vocab.update(self.added_tokens_encoder)
        return vocab

    def _tokenize(self, text):
        
        char_tokens = []
        for s in text:
            char_tokens.extend(s)
        return char_tokens

    def _convert_token_to_id(self, token):
        
        return self.vocab.get(token, self.vocab.get(self.unk_token))

    def _convert_id_to_token(self, index):
        
        return self.decoder.get(index)

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not os.path.isdir(save_directory):
            logger.error("Vocabulary path ({}) should be a directory".format(save_directory))
            return
        vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        with open(vocab_file, "w", encoding="utf-8") as f:
            f.write(json.dumps(self.vocab, indent=2, sort_keys=True, ensure_ascii=False) + "\n")

        return (vocab_file,)