

"""simple docstring"""


import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


UpperCAmelCase													=logging.get_logger(__name__)

UpperCAmelCase													={"vocab_file": "vocab.txt", "emoji_file": "emoji.json"}

UpperCAmelCase													={
    "vocab_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt",
    },
    "emoji_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json",
    },
}

UpperCAmelCase													={
    "abeja/gpt-neox-japanese-2.7b": 2_048,
}



def   _A					(	_a	:						str     ,   _a	:						List[Any]				):



      """simple docstring"""






      with open(_a     ,   """r"""     ,   encoding="""utf-8"""				) as f:
            A				  =		json.loads(f.read()				)

      A				  =		collections.OrderedDict()
      A				  =		collections.OrderedDict()
      A				  =		collections.OrderedDict()
      with open(_a     ,   """r"""     ,   encoding="""utf-8"""				) as f:
            A				  =		f.readlines()
      A				  =		[[t.rstrip("""\n"""				)] if (t == """,""" or """,""" not in t) else t.rstrip("""\n"""				).split(""","""				) for t in token]
      for idx, b in enumerate(_a				):
            A				  =		b
            A				  =		idx
            for wd in b:
                  A				  =		idx

      return vocab, raw_vocab, ids_to_tokens, emoji





class lowerCamelCase__   (				SCREAMING_SNAKE_CASE    ):



    '''simple docstring'''




    _lowerCamelCase					=   VOCAB_FILES_NAMES
    _lowerCamelCase					=   PRETRAINED_VOCAB_FILES_MAP
    _lowerCamelCase					=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _lowerCamelCase					=   ['''input_ids''', '''attention_mask''']




    def __init__( self ,lowerCamelCase_ ,lowerCamelCase_ ,lowerCamelCase_="<|endoftext|>" ,lowerCamelCase_="<|endoftext|>" ,lowerCamelCase_="<|startoftext|>" ,lowerCamelCase_="<|endoftext|>" ,lowerCamelCase_=False ,**lowerCamelCase_ ,)    ->      Optional[int]:
          super().__init__(
              unk_token=lowerCamelCase_ ,pad_token=lowerCamelCase_ ,bos_token=lowerCamelCase_ ,eos_token=lowerCamelCase_ ,do_clean_text=lowerCamelCase_ ,**lowerCamelCase_ ,)
          if not os.path.isfile(lowerCamelCase_    ):
                raise ValueError(
                    f'Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'
                    """ model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""    )
          if not os.path.isfile(lowerCamelCase_    ):
                raise ValueError(
                    f'Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'
                    """ pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""    )
          A				  =		do_clean_text
          A  ,			A  ,			A  ,			A				  =		load_vocab_and_emoji(lowerCamelCase_ ,lowerCamelCase_    )
          A				  =		SubWordJapaneseTokenizer(
              vocab=self.vocab ,ids_to_tokens=self.ids_to_tokens ,emoji=self.emoji    )




    @property
    def UpperCamelCase__		( self    )    ->      Union[str, Any]:
          # self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
          return len(self.raw_vocab    )




    def UpperCamelCase__		( self    )    ->      Dict:
          return dict(self.raw_vocab ,**self.added_tokens_encoder    )




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      int:
          return self.subword_tokenizer.tokenize(lowerCamelCase_ ,clean=self.do_clean_text    )




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      Union[str, Any]:
          return self.vocab.get(lowerCamelCase_ ,self.vocab.get(self.unk_token    )    )




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      Tuple:
          return self.subword_tokenizer.convert_id_to_token(lowerCamelCase_    )




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      str:
          A				  =		"""""".join(lowerCamelCase_    ).strip()
          return out_string




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      List[int]:
          A				  =		[]
          for is_user, text in conversation.iter_texts():
                input_ids.extend(self.encode(lowerCamelCase_ ,add_special_tokens=lowerCamelCase_    ) + [self.eos_token_id]    )

          if len(lowerCamelCase_    ) > self.model_max_length:
                A				  =		input_ids[-self.model_max_length :]
          return input_ids




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_ = None    )    ->      Tuple[str]:
          A				  =		0
          if os.path.isdir(lowerCamelCase_    ):
                A				  =		os.path.join(
                    lowerCamelCase_ ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )
                A				  =		os.path.join(
                    lowerCamelCase_ ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""emoji_file"""]    )
          else:
                A				  =		(
                    (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""vocab_file"""]
                )
                A				  =		(
                    (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""emoji_file"""]
                )
          with open(lowerCamelCase_ ,"""w""" ,encoding="""utf-8"""    ) as writer:
                for token_index, token in self.ids_to_tokens.items():
                      if index != token_index:
                            logger.warning(
                                f'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
                                """ Please check that the vocabulary is not corrupted!"""    )
                            A				  =		token_index
                      writer.write(""",""".join(lowerCamelCase_    ) + """\n"""    )
                      index += 1
          with open(lowerCamelCase_ ,"""w""" ,encoding="""utf-8"""    ) as writer:
                json.dump(self.emoji ,lowerCamelCase_    )
          return vocab_file, emoji_file





class lowerCamelCase__   (				SCREAMING_SNAKE_CASE    ):



    '''simple docstring'''




    def __init__( self ,lowerCamelCase_ ,lowerCamelCase_ ,lowerCamelCase_    )    ->      List[str]:
          A				  =		vocab  # same as swe
          A				  =		ids_to_tokens  # same as bpe
          A				  =		emoji
          A				  =		np.max([len(lowerCamelCase_    ) for w in self.vocab.keys()]    )
          A				  =		re.compile(r"""(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"""    )
          A				  =		re.compile(r"""[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"""    )
          A				  =		re.compile(r"""[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"""    )
          A				  =		re.compile(
              r"""([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""    )
          A				  =		re.compile(
              r"""(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""    )
          A				  =		re.compile(
              r"""((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"""    )
          A				  =		"""─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"""
          A				  =		"""▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"""
          A				  =		str.maketrans({k: """<BLOCK>""" for k in keisen + blocks}    )




    def __len__( self    )    ->      Any:
          return len(self.ids_to_tokens    )




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      Optional[Any]:
          A				  =		self.content_repattera.sub("""<URL>""" ,lowerCamelCase_    )
          A				  =		self.content_repattera.sub("""<EMAIL>""" ,lowerCamelCase_    )
          A				  =		self.content_repattera.sub("""<TEL>""" ,lowerCamelCase_    )
          A				  =		self.content_repattera.sub("""<DATE>""" ,lowerCamelCase_    )
          A				  =		self.content_repattera.sub("""<DATE>""" ,lowerCamelCase_    )
          A				  =		self.content_repattera.sub("""<PRICE>""" ,lowerCamelCase_    )
          A				  =		content.translate(self.content_transa    )
          while "<BLOCK><BLOCK>" in content:
                A				  =		content.replace("""<BLOCK><BLOCK>""" ,"""<BLOCK>"""    )
          return content




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_=False    )    ->      Union[str, Any]:
          A				  =		text.replace(""" """ ,"""<SP>"""    )
          A				  =		text.replace("""　""" ,"""<SP>"""    )
          A				  =		text.replace("""\r\n""" ,"""<BR>"""    )
          A				  =		text.replace("""\n""" ,"""<BR>"""    )
          A				  =		text.replace("""\r""" ,"""<BR>"""    )
          A				  =		text.replace("""\t""" ,"""<TAB>"""    )
          A				  =		text.replace("""—""" ,"""ー"""    )
          A				  =		text.replace("""−""" ,"""ー"""    )
          for k, v in self.emoji["emoji"].items():
                if k in text:
                      A				  =		text.replace(lowerCamelCase_ ,lowerCamelCase_    )
          if clean:
                A				  =		self.clean_text(lowerCamelCase_    )

          def check_simbol(lowerCamelCase_    ):
                A				  =		x.encode()
                if len(lowerCamelCase_    ) == 1 and len(lowerCamelCase_    ) == 2:
                      A				  =		(int(e[0]    ) << 8) + int(e[1]    )
                      if (
                          (c >= 0xC2A1 and c <= 0xC2BF)
                          or (c >= 0xC780 and c <= 0xC783)
                          or (c >= 0xCAB9 and c <= 0xCBBF)
                          or (c >= 0xCC80 and c <= 0xCDA2)
                      ):
                            return True
                return False

          def checkuae(lowerCamelCase_    ):
                A				  =		x.encode()
                if len(lowerCamelCase_    ) == 1 and len(lowerCamelCase_    ) == 3:
                      A				  =		(int(e[0]    ) << 1_6) + (int(e[1]    ) << 8) + int(e[2]    )
                      if c >= 0xE28080 and c <= 0xE2B07F:
                            return True
                return False

          A				  =		0
          A				  =		[]
          while pos < len(lowerCamelCase_    ):
                A				  =		min(len(lowerCamelCase_    ) ,pos + self.maxlen + 1    ) if text[pos] == """<""" else pos + 3
                A				  =		[]  # (token_id, token, pos)
                for e in range(lowerCamelCase_ ,lowerCamelCase_ ,-1    ):
                      A				  =		text[pos:e]
                      if wd in self.vocab:
                            if wd[0] == "<" and len(lowerCamelCase_    ) > 2:
                                  A				  =		[(self.vocab[wd], wd, e)]
                                  break
                            else:
                                  candidates.append((self.vocab[wd], wd, e)    )
                if len(lowerCamelCase_    ) > 0:
                      # the smallest token_id is adopted
                      A  ,			A  ,			A				  =		sorted(lowerCamelCase_ ,key=lambda lowerCamelCase_    : x[0]    )[0]
                      result.append(lowerCamelCase_    )
                      A				  =		e
                else:
                      A				  =		pos + 1
                      A				  =		text[pos:end]
                      if check_simbol(lowerCamelCase_    ):
                            result.append("""<KIGOU>"""    )
                      elif checkuae(lowerCamelCase_    ):
                            result.append("""<U2000U2BFF>"""    )
                      else:
                            for i in wd.encode("""utf-8"""    ):
                                  result.append("""<|byte%d|>""" % i    )
                      A				  =		end
          return result




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_="\n"    )    ->      Optional[int]:
          A				  =		[]
          A				  =		[]
          A				  =		self.ids_to_tokens[index][0]
          if word[:6] == "<|byte" and word[-2:] == "|>":
                byte_tokens.append(int(word[6:-2]    )    )
          else:
                if len(lowerCamelCase_    ) > 0:
                      words.append(bytearray(lowerCamelCase_    ).decode("""utf-8""" ,errors="""replace"""    )    )
                      A				  =		[]
                if word[:7] == "<|emoji" and word[-2:] == "|>":
                      words.append(self.emoji["""emoji_inv"""][word]    )
                elif word == "<SP>":
                      words.append(""" """    )
                elif word == "<BR>":
                      words.append(lowerCamelCase_    )
                elif word == "<TAB>":
                      words.append("""\t"""    )
                elif word == "<BLOCK>":
                      words.append("""▀"""    )
                elif word == "<KIGOU>":
                      words.append("""ǀ"""    )
                elif word == "<U2000U2BFF>":
                      words.append("""‖"""    )
                else:
                      words.append(lowerCamelCase_    )
          if len(lowerCamelCase_    ) > 0:
                words.append(bytearray(lowerCamelCase_    ).decode("""utf-8""" ,errors="""replace"""    )    )
          A				  =		"""""".join(lowerCamelCase_    )
          return text







"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


UpperCAmelCase													={
    "configuration_mega": ["MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP", "MegaConfig", "MegaOnnxConfig"],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase													=[
      "MEGA_PRETRAINED_MODEL_ARCHIVE_LIST",
      "MegaForCausalLM",
      "MegaForMaskedLM",
      "MegaForMultipleChoice",
      "MegaForQuestionAnswering",
      "MegaForSequenceClassification",
      "MegaForTokenClassification",
      "MegaModel",
      "MegaPreTrainedModel",
  ]

if TYPE_CHECKING:
  from .configuration_mega import MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP, MegaConfig, MegaOnnxConfig

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_mega import (
        MEGA_PRETRAINED_MODEL_ARCHIVE_LIST,
        MegaForCausalLM,
        MegaForMaskedLM,
        MegaForMultipleChoice,
        MegaForQuestionAnswering,
        MegaForSequenceClassification,
        MegaForTokenClassification,
        MegaModel,
        MegaPreTrainedModel,
    )

else:
  import sys

  UpperCAmelCase													=_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)




