codingutf8 2018 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license convert slow tokenizers checkpoints in fast serialization format of the tokenizers library import argparse import os import transformers from convertslowtokenizer import slowtofastconverters from utils import logging logging setverbosityinfo logger logging getloggername tokenizerclasses name getattrtransformers name fast for name in slowtofastconverters def convertslowcheckpointtofasttokenizername checkpointname dumppath forcedownload if tokenizername is not none and tokenizername not in tokenizerclasses raise valueerrorfunrecognized tokenizer name should be one of listtokenizerclasses keys if tokenizername is none tokenizernames tokenizerclasses else tokenizernames tokenizername getattrtransformers tokenizername fast logger infofloading tokenizer classes tokenizernames for tokenizername in tokenizernames tokenizerclass tokenizerclassestokenizername addprefix true if checkpointname is none checkpointnames listtokenizerclass maxmodelinputsizes keys else checkpointnames checkpointname logger infoffor tokenizer tokenizerclass class name loading checkpoints checkpointnames for checkpoint in checkpointnames logger infofloading tokenizerclass class name checkpoint load tokenizer tokenizer tokenizerclass frompretrainedcheckpoint forcedownloadforcedownload save fast tokenizer logger infofsave fast tokenizer to dumppath with prefix checkpoint addprefix addprefix for organization names we create subdirectories if in checkpoint checkpointdirectory checkpointprefixname checkpoint split dumppathfull os path joindumppath checkpointdirectory elif addprefix checkpointprefixname checkpoint dumppathfull dumppath else checkpointprefixname none dumppathfull dumppath logger infof dumppathfull with prefix checkpointprefixname addprefix addprefix if checkpoint in listtokenizer pretrainedvocabfilesmap values0 filepath listtokenizer pretrainedvocabfilesmap values0checkpoint nextchar filepath splitcheckpoint10 if nextchar dumppathfull os path joindumppathfull checkpointprefixname checkpointprefixname none logger infof dumppathfull with prefix checkpointprefixname addprefix addprefix filenames tokenizer savepretrained dumppathfull legacyformatfalse filenameprefixcheckpointprefixname logger infof file names filenames for filename in filenames if not filename endswithtokenizer json os removefilename logger infof removing filename if name main parser argparse argumentparser required parameters parser addargument dumppath defaultnone typestr requiredtrue helppath to output generated fast tokenizer files parser addargument tokenizername defaultnone typestr help foptional tokenizer type selected in the list of listtokenizerclasses keys if not given will download and convert all the checkpoints from aws parser addargument checkpointname defaultnone typestr helpoptional checkpoint name if not given will download and convert the canonical checkpoints from aws parser addargument forcedownload actionstoretrue helpredownload checkpoints args parser parseargs convertslowcheckpointtofastargs tokenizername args checkpointname args dumppath args forcedownload coding utf 8 2018 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license convert slow tokenizers checkpoints in fast serialization format of the tokenizers library load tokenizer save fast tokenizer for organization names we create sub directories required parameters
import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

logger = logging.get_logger(__name__)


TOKENIZER_CLASSES = {name: getattr(transformers, name + "Fast") for name in SLOW_TO_FAST_CONVERTERS}


def convert_slow_checkpoint_to_fast(tokenizer_name, checkpoint_name, dump_path, force_download):
    if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
        raise ValueError(f"Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys())}.")

    if tokenizer_name is None:
        tokenizer_names = TOKENIZER_CLASSES
    else:
        tokenizer_names = {tokenizer_name: getattr(transformers, tokenizer_name + "Fast")}

    logger.info(f"Loading tokenizer classes: {tokenizer_names}")

    for tokenizer_name in tokenizer_names:
        tokenizer_class = TOKENIZER_CLASSES[tokenizer_name]

        add_prefix = True
        if checkpoint_name is None:
            checkpoint_names = list(tokenizer_class.max_model_input_sizes.keys())
        else:
            checkpoint_names = [checkpoint_name]

        logger.info(f"For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}")

        for checkpoint in checkpoint_names:
            logger.info(f"Loading {tokenizer_class.__class__.__name__} {checkpoint}")

            
            tokenizer = tokenizer_class.from_pretrained(checkpoint, force_download=force_download)

            
            logger.info(f"Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}")

            
            if "/" in checkpoint:
                checkpoint_directory, checkpoint_prefix_name = checkpoint.split("/")
                dump_path_full = os.path.join(dump_path, checkpoint_directory)
            elif add_prefix:
                checkpoint_prefix_name = checkpoint
                dump_path_full = dump_path
            else:
                checkpoint_prefix_name = None
                dump_path_full = dump_path

            logger.info(f"=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}")

            if checkpoint in list(tokenizer.pretrained_vocab_files_map.values())[0]:
                file_path = list(tokenizer.pretrained_vocab_files_map.values())[0][checkpoint]
                next_char = file_path.split(checkpoint)[-1][0]
                if next_char == "/":
                    dump_path_full = os.path.join(dump_path_full, checkpoint_prefix_name)
                    checkpoint_prefix_name = None

                logger.info(f"=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}")

            file_names = tokenizer.save_pretrained(
                dump_path_full, legacy_format=False, filename_prefix=checkpoint_prefix_name
            )
            logger.info(f"=> File names {file_names}")

            for file_name in file_names:
                if not file_name.endswith("tokenizer.json"):
                    os.remove(file_name)
                    logger.info(f"=> removing {file_name}")


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    
    parser.add_argument(
        "--dump_path", default=None, type=str, required=True, help="Path to output generated fast tokenizer files."
    )
    parser.add_argument(
        "--tokenizer_name",
        default=None,
        type=str,
        help=(
            f"Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will "
            "download and convert all the checkpoints from AWS."
        ),
    )
    parser.add_argument(
        "--checkpoint_name",
        default=None,
        type=str,
        help="Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.",
    )
    parser.add_argument(
        "--force_download",
        action="store_true",
        help="Re-download checkpoints.",
    )
    args = parser.parse_args()

    convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)