codingutf8 2022 the salesforce s the open ai team s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for openai gpt import json import re from typing import typechecking list optional tuple union import numpy as np from utils import istfavailable istorchavailable logging if typechecking if istorchavailable import torch if istfavailable import tensorflow as tf from tokenizers import pretokenizers from tokenizationutilsbase import batchencoding from tokenizationutilsfast import pretrainedtokenizerfast from tokenizationcodegen import codegentokenizer logger logging getloggername vocabfilesnames vocabfile vocab json mergesfile merges txt tokenizerfile tokenizer json pretrainedvocabfilesmap vocabfile salesforcecodegen350mmono https huggingface cosalesforcecodegen350mmonoresolvemainvocab json mergesfile salesforcecodegen350mmono https huggingface cosalesforcecodegen350mmonoresolvemainmerges txt tokenizerfile salesforcecodegen350mmono https huggingface cosalesforcecodegen350mmonoresolvemaintokenizer json pretrainedpositionalembeddingssizes salesforcecodegen350mmono 2048 class codegentokenizerfastpretrainedtokenizerfast vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask slowtokenizerclass codegentokenizer def init self vocabfilenone mergesfilenone tokenizerfilenone unktokenendoftext bostokenendoftext eostokenendoftext addprefixspacefalse kwargs super init vocabfile mergesfile tokenizerfiletokenizerfile unktokenunktoken bostokenbostoken eostokeneostoken addprefixspaceaddprefixspace kwargs if kwargs popaddbostoken false modelid kwargs popnameorpath raise valueerror currenty gpt2 s fast tokenizer does not support adding a bos token instead you should use gpt2 s slow tokenizer class codegentokenizer as follows n fcodegentokenizer frompretrained modelid norn fautotokenizer frompretrained modelid usefastfalsen this issue will be fixed soon see https github comhuggingfacetokenizerspull1005 so that the fast tokenizer works correctly pretokstate json loadsself backendtokenizer pretokenizer getstate if pretokstate getaddprefixspace addprefixspace addprefixspace pretokclass getattrpretokenizers pretokstate poptype pretokstateaddprefixspace addprefixspace self backendtokenizer pretokenizer pretokclasspretokstate self addprefixspace addprefixspace def batchencodeplusself args kwargs batchencoding issplitintowords kwargs getissplitintowords false assert self addprefixspace or not issplitintowords fyou need to instantiate self class name with addprefixspacetrue to use it with pretokenized inputs return super batchencodeplusargs kwargs def encodeplusself args kwargs batchencoding issplitintowords kwargs getissplitintowords false assert self addprefixspace or not issplitintowords fyou need to instantiate self class name with addprefixspacetrue to use it with pretokenized inputs return super encodeplusargs kwargs def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr files self tokenizer model savesavedirectory namefilenameprefix return tuplefiles def decode self tokenids unionint listint np ndarray torch tensor tf tensor skipspecialtokens bool false cleanuptokenizationspaces bool none truncatebeforepattern optionalliststr none kwargs str decodedtext super decode tokenidstokenids skipspecialtokensskipspecialtokens cleanuptokenizationspacescleanuptokenizationspaces kwargs if truncatebeforepattern is not none and lentruncatebeforepattern 0 decodedtext self truncatedecodedtext truncatebeforepattern return decodedtext def truncateself completion truncatebeforepattern def findrestring pattern startpos m pattern searchstring startpos return m start if m else 1 terminals re compilepattern re multiline for pattern in truncatebeforepattern prints listre finditerprint completion re multiline if lenprints 1 completion completion prints1 start defs listre finditerdef completion re multiline if lendefs 1 completion completion defs1 start startpos 0 terminalspos pos for pos in findrecompletion terminal startpos for terminal in terminals if pos 1 if lenterminalspos 0 return completion minterminalspos else return completion coding utf 8 2022 the salesforce s the open ai team s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for openai gpt construct a fast codegen tokenizer backed by huggingface s tokenizers library based on byte level byte pair encoding this tokenizer has been trained to treat spaces like parts of the tokens a bit like sentencepiece so a word will be encoded differently whether it is at the beginning of the sentence without space or not python from transformers import codegentokenizerfast tokenizer codegentokenizerfast from_pretrained salesforce codegen 350m mono tokenizer hello world input_ids 15496 995 tokenizer hello world input_ids 18435 995 you can get around that behavior by passing add_prefix_space true when instantiating this tokenizer but since the model was not pretrained this way it might yield a decrease in performance tip when used with is_split_into_words true this tokenizer needs to be instantiated with add_prefix_space true tip this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str optional path to the vocabulary file merges_file str optional path to the merges file tokenizer_file str optional path to tokenizers https github com huggingface tokenizers file generally has a json extension that contains everything needed to load the tokenizer unk_token str optional defaults to endoftext the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead bos_token str optional defaults to endoftext the beginning of sequence token eos_token str optional defaults to endoftext the end of sequence token add_prefix_space bool optional defaults to false whether or not to add an initial space to the input this allows to treat the leading word just as any other word codegen tokenizer detect beginning of words by the preceding space converts a sequence of ids in a string using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces similar to doing self convert_tokens_to_string self convert_ids_to_tokens token_ids args token_ids union int list int np ndarray torch tensor tf tensor list of tokenized input ids can be obtained using the __call__ method skip_special_tokens bool optional defaults to false whether or not to remove special tokens in the decoding clean_up_tokenization_spaces bool optional whether or not to clean up the tokenization spaces if none will default to self clean_up_tokenization_spaces available in the tokenizer_config truncate_before_pattern list str optional defaults to none a list of regular expression strings that will be used to truncate the returned string this can be used to remove extra pieces of code e g truncate if observing a comment symbol at the beginning of a new line an example pattern could be re escape endoftext n n n kwargs additional keyword arguments optional will be passed to the underlying model specific decode method returns str the decoded sentence
import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
    if is_torch_available():
        import torch
    if is_tf_available():
        import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "Salesforce/codegen-350M-mono": 2048,
}


class CodeGenTokenizerFast(PreTrainedTokenizerFast):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]
    slow_tokenizer_class = CodeGenTokenizer

    def __init__(
        self,
        vocab_file=None,
        merges_file=None,
        tokenizer_file=None,
        unk_token="<|endoftext|>",
        bos_token="<|endoftext|>",
        eos_token="<|endoftext|>",
        add_prefix_space=False,
        **kwargs,
    ):
        super().__init__(
            vocab_file,
            merges_file,
            tokenizer_file=tokenizer_file,
            unk_token=unk_token,
            bos_token=bos_token,
            eos_token=eos_token,
            add_prefix_space=add_prefix_space,
            **kwargs,
        )

        if kwargs.pop("add_bos_token", False):
            model_id = kwargs.pop("name_or_path", "")
            raise ValueError(
                "Currenty GPT2's fast tokenizer does NOT support adding a BOS token. "
                "Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"
                f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
                f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
                "This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."
                " so that the fast tokenizer works correctly."
            )

        pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
        if pre_tok_state.get("add_prefix_space", add_prefix_space) != add_prefix_space:
            pre_tok_class = getattr(pre_tokenizers, pre_tok_state.pop("type"))
            pre_tok_state["add_prefix_space"] = add_prefix_space
            self.backend_tokenizer.pre_tokenizer = pre_tok_class(**pre_tok_state)

        self.add_prefix_space = add_prefix_space

    def _batch_encode_plus(self, *args, **kwargs) -> BatchEncoding:
        is_split_into_words = kwargs.get("is_split_into_words", False)
        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*args, **kwargs)

    def _encode_plus(self, *args, **kwargs) -> BatchEncoding:
        is_split_into_words = kwargs.get("is_split_into_words", False)

        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*args, **kwargs)

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
        return tuple(files)

    def decode(
        self,
        token_ids: Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"],
        skip_special_tokens: bool = False,
        clean_up_tokenization_spaces: bool = None,
        truncate_before_pattern: Optional[List[str]] = None,
        **kwargs,
    ) -> str:
        

        decoded_text = super().decode(
            token_ids=token_ids,
            skip_special_tokens=skip_special_tokens,
            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
            **kwargs,
        )

        if truncate_before_pattern is not None and len(truncate_before_pattern) > 0:
            decoded_text = self.truncate(decoded_text, truncate_before_pattern)

        return decoded_text

    def truncate(self, completion, truncate_before_pattern):
        def find_re(string, pattern, start_pos):
            m = pattern.search(string, start_pos)
            return m.start() if m else -1

        terminals = [re.compile(pattern, re.MULTILINE) for pattern in truncate_before_pattern]

        prints = list(re.finditer("^print", completion, re.MULTILINE))

        if len(prints) > 1:
            completion = completion[: prints[1].start()]

        defs = list(re.finditer("^def", completion, re.MULTILINE))

        if len(defs) > 1:
            completion = completion[: defs[1].start()]

        start_pos = 0

        terminals_pos = [
            pos for pos in [find_re(completion, terminal, start_pos) for terminal in terminals] if pos != -1
        ]

        if len(terminals_pos) > 0:
            return completion[: min(terminals_pos)]
        else:
            return completion