




import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing


class 				UpperCamelCase_    ( UpperCAmelCase__     ):

    '''simple docstring'''






    def __init__(			self      :	List[str]					, UpperCAmelCase__      :	str = "▁"					, UpperCAmelCase__      :	bool = True					, UpperCAmelCase__      :	Union[str, AddedToken] = "<unk>"					, UpperCAmelCase__      :	Union[str, AddedToken] = "</s>"					, UpperCAmelCase__      :	Union[str, AddedToken] = "<pad>"					, )						->Optional[Any]:





           '''simple docstring'''

           A__       =    {
               '''pad''': {'''id''': 0, '''token''': pad_token},
               '''eos''': {'''id''': 1, '''token''': eos_token},
               '''unk''': {'''id''': 2, '''token''': unk_token},
           }

           A__       =    [None] * len(self.special_tokens)
           for token_dict in self.special_tokens.values():
                  A__       =    token_dict['''token''']

           A__       =    Tokenizer(Unigram())

           A__       =    normalizers.Sequence(
               [
                   normalizers.Nmt(),
                   normalizers.NFKC(),
                   normalizers.Replace(Regex(''' {2,}''')					, ''' '''),
                   normalizers.Lowercase(),
               ])
           A__       =    pre_tokenizers.Sequence(
               [
                   pre_tokenizers.Metaspace(replacement=UpperCAmelCase__					, add_prefix_space=UpperCAmelCase__),
                   pre_tokenizers.Digits(individual_digits=UpperCAmelCase__),
                   pre_tokenizers.Punctuation(),
               ])
           A__       =    decoders.Metaspace(replacement=UpperCAmelCase__					, add_prefix_space=UpperCAmelCase__)

           A__       =    TemplateProcessing(
               single=f"""$A {self.special_tokens["eos"]["token"]}"""					, special_tokens=[(self.special_tokens['''eos''']['''token'''], self.special_tokens['''eos''']['''id'''])]					, )

           A__       =    {
               '''model''': '''SentencePieceUnigram''',
               '''replacement''': replacement,
               '''add_prefix_space''': add_prefix_space,
           }

           super().__init__(UpperCAmelCase__					, UpperCAmelCase__)


    def        SCREAMING_SNAKE_CASE							(			self      :	Any					, UpperCAmelCase__      :	Union[str, List[str]]					, UpperCAmelCase__      :	int = 8_000					, UpperCAmelCase__      :	bool = True					, )						->Tuple:





           '''simple docstring'''

           A__       =    trainers.UnigramTrainer(
               vocab_size=UpperCAmelCase__					, special_tokens=self.special_tokens_list					, show_progress=UpperCAmelCase__					, )

           if isinstance(UpperCAmelCase__					, UpperCAmelCase__):
                  A__       =    [files]
           self._tokenizer.train(UpperCAmelCase__					, trainer=UpperCAmelCase__)

           self.add_unk_id()


    def        SCREAMING_SNAKE_CASE							(			self      :	Tuple					, UpperCAmelCase__      :	Union[Iterator[str], Iterator[Iterator[str]]]					, UpperCAmelCase__      :	int = 8_000					, UpperCAmelCase__      :	bool = True					, )						->List[Any]:





           '''simple docstring'''

           A__       =    trainers.UnigramTrainer(
               vocab_size=UpperCAmelCase__					, special_tokens=self.special_tokens_list					, show_progress=UpperCAmelCase__					, )

           self._tokenizer.train_from_iterator(UpperCAmelCase__					, trainer=UpperCAmelCase__)

           self.add_unk_id()





    def        SCREAMING_SNAKE_CASE							(			self      :	Any)						->str:





           '''simple docstring'''

           A__       =    json.loads(self._tokenizer.to_str())

           A__       =    self.special_tokens['''unk''']['''id''']

           A__       =    Tokenizer.from_str(json.dumps(UpperCAmelCase__))





'''simple docstring'''






import os

lowerCAmelCase__											=	{'''I''': 1, '''V''': 5, '''X''': 10, '''L''': 50, '''C''': 100, '''D''': 500, '''M''': 1000}


def 		_A     (    A__      ):






 """simple docstring"""



 __lowercase							  = 0

 __lowercase							  = 0
 while index < len(A__      ) - 1:
  __lowercase							  = SYMBOLS[numerals[index]]
  __lowercase							  = SYMBOLS[numerals[index + 1]]
  if current_value < next_value:
   total_value -= current_value
  else:
   total_value += current_value
  index += 1
 total_value += SYMBOLS[numerals[index]]

 return total_value


def 		_A     (    A__      ):






 """simple docstring"""



 __lowercase							  = ''''''

 __lowercase							  = num // 1000
 numerals += m_count * "M"
 num %= 1000

 __lowercase							  = num // 100
 if c_count == 9:
  numerals += "CM"
  c_count -= 9
 elif c_count == 4:
  numerals += "CD"
  c_count -= 4
 if c_count >= 5:
  numerals += "D"
  c_count -= 5
 numerals += c_count * "C"
 num %= 100

 __lowercase							  = num // 10
 if x_count == 9:
  numerals += "XC"
  x_count -= 9
 elif x_count == 4:
  numerals += "XL"
  x_count -= 4
 if x_count >= 5:
  numerals += "L"
  x_count -= 5
 numerals += x_count * "X"
 num %= 10

 if num == 9:
  numerals += "IX"
  num -= 9
 elif num == 4:
  numerals += "IV"
  num -= 4
 if num >= 5:
  numerals += "V"
  num -= 5
 numerals += num * "I"

 return numerals


def 		_A     (    A__ = "/p089_roman.txt"      ):






 """simple docstring"""



 __lowercase							  = 0

 with open(os.path.dirname(A__      ) + roman_numerals_filename      ) as filea:
  __lowercase							  = filea.readlines()

 for line in lines:
  __lowercase							  = line.strip()
  __lowercase							  = parse_roman_numerals(A__      )
  __lowercase							  = generate_roman_numerals(A__      )
  savings += len(A__      ) - len(A__      )

 return savings


if __name__ == "__main__":
 print(f'{solution() = }')

