import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCamelCase  							=						re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

UpperCamelCase  							=						None

def 			lowercase_				(			):
    lowercase__ :   List[Any]    					=				argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0.")
    parser.add_argument("data_file"		,	metavar="data.json"		,	help="Input data JSON file.")
    parser.add_argument("pred_file"		,	metavar="pred.json"		,	help="Model predictions.")
    parser.add_argument(
        "--out-file"		,	"-o"		,	metavar="eval.json"		,	help="Write accuracy metrics to file (default is stdout).")
    parser.add_argument(
        "--na-prob-file"		,	"-n"		,	metavar="na_prob.json"		,	help="Model estimates of probability of no answer.")
    parser.add_argument(
        "--na-prob-thresh"		,	"-t"		,	type=_lowerCamelCase		,	default=1.0		,	help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."		,	)
    parser.add_argument(
        "--out-image-dir"		,	"-p"		,	metavar="out_images"		,	default=_lowerCamelCase		,	help="Save precision-recall curves to directory.")
    parser.add_argument("--verbose"		,	"-v"		,	action="store_true")
    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)
    return parser.parse_args()

def 			lowercase_				(			_lowerCamelCase							:       List[Any]):
    lowercase__ :   Union[str, Any]    					=				{}
    for article in dataset:
        for p in article["paragraphs"]:
            for qa in p["qas"]:
                lowercase__ :   Optional[int]    					=				bool(qa["answers"]["text"])
    return qid_to_has_ans

def 			lowercase_				(			_lowerCamelCase							:       Union[str, Any]):

    def remove_articles(_lowerCamelCase							:       List[str]):
        return ARTICLES_REGEX.sub(" "		,	_lowerCamelCase)

    def white_space_fix(_lowerCamelCase							:       int):
        return " ".join(text.split())

    def remove_punc(_lowerCamelCase							:       Tuple):
        lowercase__ :   Any    					=				set(string.punctuation)
        return "".join(ch for ch in text if ch not in exclude)

    def lower(_lowerCamelCase							:       Any):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(_lowerCamelCase))))

def 			lowercase_				(			_lowerCamelCase							:       str):
    if not s:
        return []
    return normalize_answer(_lowerCamelCase).split()

def 			lowercase_				(			_lowerCamelCase							:       Optional[int]		,	_lowerCamelCase							:       Any):
    return int(normalize_answer(_lowerCamelCase) == normalize_answer(_lowerCamelCase))

def 			lowercase_				(			_lowerCamelCase							:       Union[str, Any]		,	_lowerCamelCase							:       Optional[Any]):
    lowercase__ :   str    					=				get_tokens(_lowerCamelCase)
    lowercase__ :   int    					=				get_tokens(_lowerCamelCase)
    lowercase__ :   List[Any]    					=				collections.Counter(_lowerCamelCase) & collections.Counter(_lowerCamelCase)
    lowercase__ :   List[Any]    					=				sum(common.values())
    if len(_lowerCamelCase) == 0 or len(_lowerCamelCase) == 0:
        # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
        return int(gold_toks == pred_toks)
    if num_same == 0:
        return 0
    lowercase__ :   Union[str, Any]    					=				1.0 * num_same / len(_lowerCamelCase)
    lowercase__ :   Optional[Any]    					=				1.0 * num_same / len(_lowerCamelCase)
    lowercase__ :   Dict    					=				(2 * precision * recall) / (precision + recall)
    return fa

def 			lowercase_				(			_lowerCamelCase							:       Tuple		,	_lowerCamelCase							:       Any):
    lowercase__ :   Union[str, Any]    					=				{}
    lowercase__ :   List[str]    					=				{}
    for article in dataset:
        for p in article["paragraphs"]:
            for qa in p["qas"]:
                lowercase__ :   List[Any]    					=				qa["id"]
                lowercase__ :   List[Any]    					=				[t for t in qa["answers"]["text"] if normalize_answer(_lowerCamelCase)]
                if not gold_answers:
                    # For unanswerable questions, only correct answer is empty string
                    lowercase__ :   List[str]    					=				[""]
                if qid not in preds:
                    print(f'''Missing prediction for {qid}''')
                    continue
                lowercase__ :   Any    					=				preds[qid]
                # Take max over all gold answers
                lowercase__ :   Tuple    					=				max(compute_exact(_lowerCamelCase		,	_lowerCamelCase) for a in gold_answers)
                lowercase__ :   Any    					=				max(compute_fa(_lowerCamelCase		,	_lowerCamelCase) for a in gold_answers)
    return exact_scores, fa_scores

def 			lowercase_				(			_lowerCamelCase							:       Union[str, Any]		,	_lowerCamelCase							:       Dict		,	_lowerCamelCase							:       Any		,	_lowerCamelCase							:       Any):
    lowercase__ :   Union[str, Any]    					=				{}
    for qid, s in scores.items():
        lowercase__ :   Optional[int]    					=				na_probs[qid] > na_prob_thresh
        if pred_na:
            lowercase__ :   Dict    					=				float(not qid_to_has_ans[qid])
        else:
            lowercase__ :   List[Any]    					=				s
    return new_scores

def 			lowercase_				(			_lowerCamelCase							:       Dict		,	_lowerCamelCase							:       Union[str, Any]		,	_lowerCamelCase							:       Union[str, Any]=None):
    if not qid_list:
        lowercase__ :   List[str]    					=				len(_lowerCamelCase)
        return collections.OrderedDict(
            [
                ("exact", 100.0 * sum(exact_scores.values()) / total),
                ("f1", 100.0 * sum(fa_scores.values()) / total),
                ("total", total),
            ])
    else:
        lowercase__ :   Union[str, Any]    					=				len(_lowerCamelCase)
        return collections.OrderedDict(
            [
                ("exact", 100.0 * sum(exact_scores[k] for k in qid_list) / total),
                ("f1", 100.0 * sum(fa_scores[k] for k in qid_list) / total),
                ("total", total),
            ])

def 			lowercase_				(			_lowerCamelCase							:       Dict		,	_lowerCamelCase							:       Any		,	_lowerCamelCase							:       Any):
    for k in new_eval:
        lowercase__ :   Union[str, Any]    					=				new_eval[k]

def 			lowercase_				(			_lowerCamelCase							:       Optional[Any]		,	_lowerCamelCase							:       Optional[int]		,	_lowerCamelCase							:       Tuple		,	_lowerCamelCase							:       List[Any]):
    plt.step(_lowerCamelCase		,	_lowerCamelCase		,	color="b"		,	alpha=0.2		,	where="post")
    plt.fill_between(_lowerCamelCase		,	_lowerCamelCase		,	step="post"		,	alpha=0.2		,	color="b")
    plt.xlabel("Recall")
    plt.ylabel("Precision")
    plt.xlim([0.0, 1.05])
    plt.ylim([0.0, 1.05])
    plt.title(_lowerCamelCase)
    plt.savefig(_lowerCamelCase)
    plt.clf()

def 			lowercase_				(			_lowerCamelCase							:       Any		,	_lowerCamelCase							:       List[Any]		,	_lowerCamelCase							:       int		,	_lowerCamelCase							:       List[Any]		,	_lowerCamelCase							:       Dict=None		,	_lowerCamelCase							:       Optional[int]=None):
    lowercase__ :   Optional[Any]    					=				sorted(_lowerCamelCase		,	key=lambda _lowerCamelCase: na_probs[k])
    lowercase__ :   Optional[int]    					=				0.0
    lowercase__ :   str    					=				1.0
    lowercase__ :   List[str]    					=				0.0
    lowercase__ :   List[str]    					=				[1.0]
    lowercase__ :   Union[str, Any]    					=				[0.0]
    lowercase__ :   Dict    					=				0.0
    for i, qid in enumerate(_lowerCamelCase):
        if qid_to_has_ans[qid]:
            true_pos += scores[qid]
        lowercase__ :   str    					=				true_pos / float(i + 1)
        lowercase__ :   List[str]    					=				true_pos / float(_lowerCamelCase)
        if i == len(_lowerCamelCase) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
            # i.e., if we can put a threshold after this point
            avg_prec += cur_p * (cur_r - recalls[-1])
            precisions.append(_lowerCamelCase)
            recalls.append(_lowerCamelCase)
    if out_image:
        plot_pr_curve(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase)
    return {"ap": 100.0 * avg_prec}

def 			lowercase_				(			_lowerCamelCase							:       str		,	_lowerCamelCase							:       Optional[Any]		,	_lowerCamelCase							:       int		,	_lowerCamelCase							:       Tuple		,	_lowerCamelCase							:       Union[str, Any]		,	_lowerCamelCase							:       Optional[Any]):
    if out_image_dir and not os.path.exists(_lowerCamelCase):
        os.makedirs(_lowerCamelCase)
    lowercase__ :   Union[str, Any]    					=				sum(1 for v in qid_to_has_ans.values() if v)
    if num_true_pos == 0:
        return
    lowercase__ :   Tuple    					=				make_precision_recall_eval(
        _lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	out_image=os.path.join(_lowerCamelCase		,	"pr_exact.png")		,	title="Precision-Recall curve for Exact Match score"		,	)
    lowercase__ :   Union[str, Any]    					=				make_precision_recall_eval(
        _lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	out_image=os.path.join(_lowerCamelCase		,	"pr_f1.png")		,	title="Precision-Recall curve for F1 score"		,	)
    lowercase__ :   Union[str, Any]    					=				{k: float(_lowerCamelCase) for k, v in qid_to_has_ans.items()}
    lowercase__ :   Optional[int]    					=				make_precision_recall_eval(
        _lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	out_image=os.path.join(_lowerCamelCase		,	"pr_oracle.png")		,	title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"		,	)
    merge_eval(_lowerCamelCase		,	_lowerCamelCase		,	"pr_exact")
    merge_eval(_lowerCamelCase		,	_lowerCamelCase		,	"pr_f1")
    merge_eval(_lowerCamelCase		,	_lowerCamelCase		,	"pr_oracle")

def 			lowercase_				(			_lowerCamelCase							:       Tuple		,	_lowerCamelCase							:       List[Any]		,	_lowerCamelCase							:       Optional[Any]		,	_lowerCamelCase							:       List[Any]):
    if not qid_list:
        return
    lowercase__ :   List[str]    					=				[na_probs[k] for k in qid_list]
    lowercase__ :   Tuple    					=				np.ones_like(_lowerCamelCase) / float(len(_lowerCamelCase))
    plt.hist(_lowerCamelCase		,	weights=_lowerCamelCase		,	bins=20		,	range=(0.0, 1.0))
    plt.xlabel("Model probability of no-answer")
    plt.ylabel("Proportion of dataset")
    plt.title(f'''Histogram of no-answer probability: {name}''')
    plt.savefig(os.path.join(_lowerCamelCase		,	f'''na_prob_hist_{name}.png'''))
    plt.clf()

def 			lowercase_				(			_lowerCamelCase							:       List[str]		,	_lowerCamelCase							:       Any		,	_lowerCamelCase							:       Any		,	_lowerCamelCase							:       Optional[int]):
    lowercase__ :   Dict    					=				sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k])
    lowercase__ :   Union[str, Any]    					=				num_no_ans
    lowercase__ :   Optional[int]    					=				cur_score
    lowercase__ :   List[str]    					=				0.0
    lowercase__ :   List[Any]    					=				sorted(_lowerCamelCase		,	key=lambda _lowerCamelCase: na_probs[k])
    for i, qid in enumerate(_lowerCamelCase):
        if qid not in scores:
            continue
        if qid_to_has_ans[qid]:
            lowercase__ :   List[Any]    					=				scores[qid]
        else:
            if preds[qid]:
                lowercase__ :   str    					=				-1
            else:
                lowercase__ :   List[Any]    					=				0
        cur_score += diff
        if cur_score > best_score:
            lowercase__ :   Any    					=				cur_score
            lowercase__ :   Dict    					=				na_probs[qid]
    return 100.0 * best_score / len(_lowerCamelCase), best_thresh

def 			lowercase_				(			_lowerCamelCase							:       Optional[int]		,	_lowerCamelCase							:       str		,	_lowerCamelCase							:       List[str]		,	_lowerCamelCase							:       int		,	_lowerCamelCase							:       Optional[Any]		,	_lowerCamelCase							:       Optional[Any]):
    lowercase__					,   lowercase__ :   int    					=				find_best_thresh(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase)
    lowercase__					,   lowercase__ :   List[str]    					=				find_best_thresh(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase)
    lowercase__ :   Optional[Any]    					=				best_exact
    lowercase__ :   int    					=				exact_thresh
    lowercase__ :   Tuple    					=				best_fa
    lowercase__ :   Optional[Any]    					=				fa_thresh

def 			lowercase_				(			):
    with open(OPTS.data_file) as f:
        lowercase__ :   List[Any]    					=				json.load(_lowerCamelCase)
        lowercase__ :   Union[str, Any]    					=				dataset_json["data"]
    with open(OPTS.pred_file) as f:
        lowercase__ :   Optional[int]    					=				json.load(_lowerCamelCase)
    if OPTS.na_prob_file:
        with open(OPTS.na_prob_file) as f:
            lowercase__ :   Union[str, Any]    					=				json.load(_lowerCamelCase)
    else:
        lowercase__ :   List[str]    					=				{k: 0.0 for k in preds}
    lowercase__ :   Tuple    					=				make_qid_to_has_ans(_lowerCamelCase)  # maps qid to True/False
    lowercase__ :   int    					=				[k for k, v in qid_to_has_ans.items() if v]
    lowercase__ :   str    					=				[k for k, v in qid_to_has_ans.items() if not v]
    lowercase__					,   lowercase__ :   Any    					=				get_raw_scores(_lowerCamelCase		,	_lowerCamelCase)
    lowercase__ :   int    					=				apply_no_ans_threshold(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	OPTS.na_prob_thresh)
    lowercase__ :   int    					=				apply_no_ans_threshold(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	OPTS.na_prob_thresh)
    lowercase__ :   Dict    					=				make_eval_dict(_lowerCamelCase		,	_lowerCamelCase)
    if has_ans_qids:
        lowercase__ :   Dict    					=				make_eval_dict(_lowerCamelCase		,	_lowerCamelCase		,	qid_list=_lowerCamelCase)
        merge_eval(_lowerCamelCase		,	_lowerCamelCase		,	"HasAns")
    if no_ans_qids:
        lowercase__ :   str    					=				make_eval_dict(_lowerCamelCase		,	_lowerCamelCase		,	qid_list=_lowerCamelCase)
        merge_eval(_lowerCamelCase		,	_lowerCamelCase		,	"NoAns")
    if OPTS.na_prob_file:
        find_all_best_thresh(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase)
    if OPTS.na_prob_file and OPTS.out_image_dir:
        run_precision_recall_analysis(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase		,	OPTS.out_image_dir)
        histogram_na_prob(_lowerCamelCase		,	_lowerCamelCase		,	OPTS.out_image_dir		,	"hasAns")
        histogram_na_prob(_lowerCamelCase		,	_lowerCamelCase		,	OPTS.out_image_dir		,	"noAns")
    if OPTS.out_file:
        with open(OPTS.out_file		,	"w") as f:
            json.dump(_lowerCamelCase		,	_lowerCamelCase)
    else:
        print(json.dumps(_lowerCamelCase		,	indent=2))


if __name__ == "__main__":
      UpperCamelCase  							=						parse_args()
      if OPTS.out_image_dir:
            import matplotlib

            matplotlib.use('''Agg''')
            import matplotlib.pyplot as plt
      main()





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case :		int							      =  logging.get_logger(__name__)
snake_case :		List[str]							      =  {"vocab_file": "spiece.model"}

snake_case :		List[str]							      =  {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

snake_case :		Tuple							      =  {
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

snake_case :		List[str]							      =  "▁"





class       _snake_case					(     snake_case				):
			UpperCamelCase__          =    VOCAB_FILES_NAMES
			UpperCamelCase__          =    PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase__          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







			def __init__(				self						,   _a						,   _a=True						,   _a=True						,   _a=False						,   _a="[CLS]"						,   _a="[SEP]"						,   _a="<unk>"						,   _a="[SEP]"						,   _a="<pad>"						,   _a="[CLS]"						,   _a="[MASK]"						,   _a = None						,   **_a						,   ):
				# Mask token behave like a normal word, i.e. include the space before it and
				# is included in the raw text, there should be a match in a non-normalized sentence.
				__magic_name__		:		str	   =     (
				    AddedToken(_a						,   lstrip=_a						,   rstrip=_a						,   normalized=_a		)
				    if isinstance(_a						,   _a		)
				    else mask_token
				)

				__magic_name__		:		Union[str, Any]	   =     {} if sp_model_kwargs is None else sp_model_kwargs

				super().__init__(
				    do_lower_case=_a						,   remove_space=_a						,   keep_accents=_a						,   bos_token=_a						,   eos_token=_a						,   unk_token=_a						,   sep_token=_a						,   pad_token=_a						,   cls_token=_a						,   mask_token=_a						,   sp_model_kwargs=self.sp_model_kwargs						,   **_a						,   )

				__magic_name__		:		Dict	   =     do_lower_case
				__magic_name__		:		Tuple	   =     remove_space
				__magic_name__		:		Union[str, Any]	   =     keep_accents
				__magic_name__		:		Tuple	   =     vocab_file

				__magic_name__		:		int	   =     spm.SentencePieceProcessor(**self.sp_model_kwargs		)
				self.sp_model.Load(_a		)







			@property
			def 			SCREAMING_SNAKE_CASE    (				self		):
				return len(self.sp_model		)







			def 			SCREAMING_SNAKE_CASE    (				self		):
				__magic_name__		:		List[str]	   =     {self.convert_ids_to_tokens(_a		): i for i in range(self.vocab_size		)}
				vocab.update(self.added_tokens_encoder		)
				return vocab







			def __getstate__(				self		):
				__magic_name__		:		List[str]	   =     self.__dict__.copy()
				__magic_name__		:		Any	   =     None
				return state







			def __setstate__(				self						,   _a		):
				__magic_name__		:		Union[str, Any]	   =     d

				# for backward compatibility
				if not hasattr(self						,   "sp_model_kwargs"		):
					__magic_name__		:		str	   =     {}

				__magic_name__		:		Optional[int]	   =     spm.SentencePieceProcessor(**self.sp_model_kwargs		)
				self.sp_model.Load(self.vocab_file		)







			def 			SCREAMING_SNAKE_CASE    (				self						,   _a		):
				if self.remove_space:
					__magic_name__		:		List[Any]	   =     " ".join(inputs.strip().split()		)
				else:
					__magic_name__		:		str	   =     inputs
				__magic_name__		:		int	   =     outputs.replace("``"						,   "\""		).replace("''"						,   "\""		)

				if not self.keep_accents:
					__magic_name__		:		str	   =     unicodedata.normalize("NFKD"						,   _a		)
					__magic_name__		:		Tuple	   =     "".join([c for c in outputs if not unicodedata.combining(_a		)]		)
				if self.do_lower_case:
					__magic_name__		:		int	   =     outputs.lower()

				return outputs







			def 			SCREAMING_SNAKE_CASE    (				self						,   _a		):
				__magic_name__		:		Optional[Any]	   =     self.preprocess_text(_a		)
				__magic_name__		:		Dict	   =     self.sp_model.encode(_a						,   out_type=_a		)
				__magic_name__		:		Any	   =     []
				for piece in pieces:
					if len(_a		) > 1 and piece[-1] == str(","		) and piece[-2].isdigit():
						__magic_name__		:		Optional[Any]	   =     self.sp_model.EncodeAsPieces(piece[:-1].replace(_a						,   ""		)		)
						if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
							if len(cur_pieces[0]		) == 1:
								__magic_name__		:		List[str]	   =     cur_pieces[1:]
							else:
								__magic_name__		:		Optional[int]	   =     cur_pieces[0][1:]
						cur_pieces.append(piece[-1]		)
						new_pieces.extend(_a		)
					else:
						new_pieces.append(_a		)

				return new_pieces







			def 			SCREAMING_SNAKE_CASE    (				self						,   _a		):
				return self.sp_model.PieceToId(_a		)







			def 			SCREAMING_SNAKE_CASE    (				self						,   _a		):
				return self.sp_model.IdToPiece(_a		)







			def 			SCREAMING_SNAKE_CASE    (				self						,   _a		):
				__magic_name__		:		Any	   =     []
				__magic_name__		:		Union[str, Any]	   =     ""
				__magic_name__		:		int	   =     False
				for token in tokens:
					# make sure that special tokens are not decoded using sentencepiece model
					if token in self.all_special_tokens:
						if not prev_is_special:
							out_string += " "
						out_string += self.sp_model.decode(_a		) + token
						__magic_name__		:		List[Any]	   =     True
						__magic_name__		:		Optional[int]	   =     []
					else:
						current_sub_tokens.append(_a		)
						__magic_name__		:		Optional[Any]	   =     False
				out_string += self.sp_model.decode(_a		)
				return out_string.strip()







			def 			SCREAMING_SNAKE_CASE    (				self						,   _a						,   _a = None		):
				__magic_name__		:		List[str]	   =     [self.sep_token_id]
				__magic_name__		:		Union[str, Any]	   =     [self.cls_token_id]
				if token_ids_a is None:
					return cls + token_ids_a + sep
				return cls + token_ids_a + sep + token_ids_a + sep







			def 			SCREAMING_SNAKE_CASE    (				self						,   _a						,   _a = None						,   _a = False		):

				if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=_a						,   token_ids_a=_a						,   already_has_special_tokens=_a		)

				if token_ids_a is not None:
					return [1] + ([0] * len(_a		)) + [1] + ([0] * len(_a		)) + [1]
				return [1] + ([0] * len(_a		)) + [1]







			def 			SCREAMING_SNAKE_CASE    (				self						,   _a						,   _a = None		):
				__magic_name__		:		Optional[int]	   =     [self.sep_token_id]
				__magic_name__		:		Union[str, Any]	   =     [self.cls_token_id]

				if token_ids_a is None:
					return len(cls + token_ids_a + sep		) * [0]
				return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







			def 			SCREAMING_SNAKE_CASE    (				self						,   _a						,   _a = None		):
				if not os.path.isdir(_a		):
					logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''		)
					return
				__magic_name__		:		List[str]	   =     os.path.join(
				    _a						,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

				if os.path.abspath(self.vocab_file		) != os.path.abspath(_a		) and os.path.isfile(self.vocab_file		):
					copyfile(self.vocab_file						,   _a		)
				elif not os.path.isfile(self.vocab_file		):
					with open(_a						,   "wb"		) as fi:
						__magic_name__		:		Union[str, Any]	   =     self.sp_model.serialized_model_proto()
						fi.write(_a		)

				return (out_vocab_file,)
