


'''simple docstring'''







import os
import re
import shutil
import sys
import tempfile
import unittest

import black


A__:    Any					    =       os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, '''utils'''))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
A__:    Union[str, Any]					    =       '''    def __init__(self, config):
        super().__init__()
        self.transform = BertPredictionHeadTransform(config)

        # The output weights are the same as the input embeddings, but there is
        # an output-only bias for each token.
        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        self.bias = nn.Parameter(torch.zeros(config.vocab_size))

        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
        self.decoder.bias = self.bias

    def forward(self, hidden_states):
        hidden_states = self.transform(hidden_states)
        hidden_states = self.decoder(hidden_states)
        return hidden_states
'''







class A__       (						unittest.TestCase      ):






      def __UpperCAmelCase       ( self			:Optional[Any]	)       ->					Optional[Any]:




             '''simple docstring'''
             _a    : str     					=tempfile.mkdtemp()
             os.makedirs(os.path.join(self.transformer_dir     ,						"""models/bert/"""	)	)
             _a    : Optional[Any]     					=self.transformer_dir
             shutil.copy(
                 os.path.join(SCREAMING_SNAKE_CASE     ,						"""src/transformers/models/bert/modeling_bert.py"""	)     ,						os.path.join(self.transformer_dir     ,						"""models/bert/modeling_bert.py"""	)     ,						)






      def __UpperCAmelCase       ( self			:List[str]	)       ->					List[Any]:




             '''simple docstring'''
             _a    : Optional[Any]     					="""src/transformers"""
             shutil.rmtree(self.transformer_dir	)






      def __UpperCAmelCase       ( self			:Union[str, Any]     ,						SCREAMING_SNAKE_CASE			:Optional[int]     ,						SCREAMING_SNAKE_CASE			:Optional[int]     ,						SCREAMING_SNAKE_CASE			:List[str]     ,						SCREAMING_SNAKE_CASE			:Optional[Any]=None	)       ->					Tuple:




             '''simple docstring'''
             _a    : List[Any]     					=comment + f"\nclass {class_name}(nn.Module):\n" + class_code
             if overwrite_result is not None:
                    _a    : Any     					=comment + f"\nclass {class_name}(nn.Module):\n" + overwrite_result
             _a    : Optional[Any]     					=black.Mode(target_versions={black.TargetVersion.PYaa}     ,						line_length=1_1_9	)
             _a    : List[Any]     					=black.format_str(SCREAMING_SNAKE_CASE     ,						mode=SCREAMING_SNAKE_CASE	)
             _a    : Optional[int]     					=os.path.join(self.transformer_dir     ,						"""new_code.py"""	)
             with open(SCREAMING_SNAKE_CASE     ,						"""w"""     ,						newline="""\n"""	) as f:
                    f.write(SCREAMING_SNAKE_CASE	)
             if overwrite_result is None:
                    self.assertTrue(len(check_copies.is_copy_consistent(SCREAMING_SNAKE_CASE	)	) == 0	)
             else:
                    check_copies.is_copy_consistent(f.name     ,						overwrite=SCREAMING_SNAKE_CASE	)
                    with open(SCREAMING_SNAKE_CASE     ,						"""r"""	) as f:
                           self.assertTrue(f.read()     ,						SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:Any	)       ->					Optional[Any]:




             '''simple docstring'''
             _a    : Dict     					=check_copies.find_code_in_transformers("""models.bert.modeling_bert.BertLMPredictionHead"""	)
             self.assertEqual(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:str	)       ->					str:




             '''simple docstring'''
             # Base copy consistency
             self.check_copy_consistency(
                 """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"""     ,						"""BertLMPredictionHead"""     ,						REFERENCE_CODE + """\n"""     ,						)

             # With no empty line at the end
             self.check_copy_consistency(
                 """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"""     ,						"""BertLMPredictionHead"""     ,						SCREAMING_SNAKE_CASE     ,						)

             # Copy consistency with rename
             self.check_copy_consistency(
                 """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"""     ,						"""TestModelLMPredictionHead"""     ,						re.sub("""Bert"""     ,						"""TestModel"""     ,						SCREAMING_SNAKE_CASE	)     ,						)

             # Copy consistency with a really long name
             _a    : int     					="""TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason"""
             self.check_copy_consistency(
                 f"# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}"     ,						f"{long_class_name}LMPredictionHead"     ,						re.sub("""Bert"""     ,						SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)     ,						)

             # Copy consistency with overwrite
             self.check_copy_consistency(
                 """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"""     ,						"""TestModelLMPredictionHead"""     ,						SCREAMING_SNAKE_CASE     ,						overwrite_result=re.sub("""Bert"""     ,						"""TestModel"""     ,						SCREAMING_SNAKE_CASE	)     ,						)






      def __UpperCAmelCase       ( self			:str	)       ->					List[str]:




             '''simple docstring'''
             _a    : List[Any]     					=check_copies.LOCALIZED_READMES["""README_zh-hans.md"""]

             _a    : Tuple     					=(
                 """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"""
                 """ Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"""
                 """ Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"""
                 """ Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1."""
                 """ **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),"""
                 """ released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"""
                 """ lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same"""
                 """ method has been applied to compress GPT2 into"""
                 """ [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"""
                 """ [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"""
                 """ Multilingual BERT into"""
                 """ [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"""
                 """ version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**"""
                 """ (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders"""
                 """ as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang"""
                 """ Luong, Quoc V. Le, Christopher D. Manning."""
             )
             _a    : Union[str, Any]     					=(
                 """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
                 """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
                 """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
                 """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
             )
             _a    : Optional[int]     					=(
                 """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
                 """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
                 """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
                 """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1."""
                 """ **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文"""
                 """ [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"""
                 """ lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same"""
                 """ method has been applied to compress GPT2 into"""
                 """ [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"""
                 """ [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"""
                 """ Multilingual BERT into"""
                 """ [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"""
                 """ version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自"""
                 """ Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather"""
                 """ than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,"""
                 """ Christopher D. Manning 发布。\n"""
             )

             _a					,    _a    : Union[str, Any]     					=check_copies.convert_to_localized_md(
                 SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE     ,						localized_readme["""format_model_list"""]	)

             self.assertFalse(SCREAMING_SNAKE_CASE	)
             self.assertEqual(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)

             _a					,    _a    : List[str]     					=check_copies.convert_to_localized_md(
                 SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE     ,						localized_readme["""format_model_list"""]	)

             # Check whether the number of models is equal to README.md after conversion.
             self.assertTrue(SCREAMING_SNAKE_CASE	)

             _a    : Dict     					=(
                 """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"""
                 """ Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"""
                 """ Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"""
                 """ Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut."""
             )
             _a    : Tuple     					=(
                 """1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and"""
                 """ the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
                 """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
                 """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
             )
             _a    : Optional[Any]     					=(
                 """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
                 """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
                 """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
                 """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
             )

             _a					,    _a    : Union[str, Any]     					=check_copies.convert_to_localized_md(
                 SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE     ,						localized_readme["""format_model_list"""]	)

             # Check if the model link is synchronized.
             self.assertEqual(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)






'''simple docstring'''







from typing import Dict, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import flip_channel_order, resize, to_channel_dimension_format, to_pil_image
from ...image_utils import (
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_pytesseract_available, is_vision_available, logging, requires_backends


if is_vision_available():
   import PIL

# soft dependency
if is_pytesseract_available():
   import pytesseract

A__:    Union[str, Any]					    =       logging.get_logger(__name__)


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	str    ,_UpperCAmelCase	:	Tuple    ,_UpperCAmelCase	:	List[str]  )  ->			int:
       return [
           int(1000 * (box[0] / width)  ),
           int(1000 * (box[1] / height)  ),
           int(1000 * (box[2] / width)  ),
           int(1000 * (box[3] / height)  ),
       ]


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	np.ndarray    ,_UpperCAmelCase	:	Optional[str]    ,_UpperCAmelCase	:	Optional[str] = None  )  ->			Optional[int]:
       _a    : Any     					=tesseract_config if tesseract_config is not None else """"""

       # apply OCR
       _a    : Optional[Any]     					=to_pil_image(_UpperCAmelCase  )
       _a					,    _a    : List[Any]     					=pil_image.size
       _a    : List[str]     					=pytesseract.image_to_data(_UpperCAmelCase    ,lang=_UpperCAmelCase    ,output_type="""dict"""    ,config=_UpperCAmelCase  )
       _a					,    _a					,    _a					,    _a					,    _a    : str     					=data["""text"""], data["""left"""], data["""top"""], data["""width"""], data["""height"""]

       # filter empty words and corresponding coordinates
       _a    : Tuple     					=[idx for idx, word in enumerate(_UpperCAmelCase  ) if not word.strip()]
       _a    : List[Any]     					=[word for idx, word in enumerate(_UpperCAmelCase  ) if idx not in irrelevant_indices]
       _a    : Dict     					=[coord for idx, coord in enumerate(_UpperCAmelCase  ) if idx not in irrelevant_indices]
       _a    : List[str]     					=[coord for idx, coord in enumerate(_UpperCAmelCase  ) if idx not in irrelevant_indices]
       _a    : Union[str, Any]     					=[coord for idx, coord in enumerate(_UpperCAmelCase  ) if idx not in irrelevant_indices]
       _a    : Union[str, Any]     					=[coord for idx, coord in enumerate(_UpperCAmelCase  ) if idx not in irrelevant_indices]

       # turn coordinates into (left, top, left+width, top+height) format
       _a    : List[str]     					=[]
       for x, y, w, h in zip(_UpperCAmelCase    ,_UpperCAmelCase    ,_UpperCAmelCase    ,_UpperCAmelCase  ):
              _a    : int     					=[x, y, x + w, y + h]
              actual_boxes.append(_UpperCAmelCase  )

       # finally, normalize the bounding boxes
       _a    : str     					=[]
       for box in actual_boxes:
              normalized_boxes.append(normalize_box(_UpperCAmelCase    ,_UpperCAmelCase    ,_UpperCAmelCase  )  )

       assert len(_UpperCAmelCase  ) == len(_UpperCAmelCase  ), "Not as many words as there are bounding boxes"

       return words, normalized_boxes







class A__       (						UpperCAmelCase__      ):
      __UpperCamelCase       :  List[Any]	       =							["pixel_values"]






      def __init__( self			:Tuple     ,						SCREAMING_SNAKE_CASE			:bool = True     ,						SCREAMING_SNAKE_CASE			:Dict[str, int] = None     ,						SCREAMING_SNAKE_CASE			:PILImageResampling = PILImageResampling.BILINEAR     ,						SCREAMING_SNAKE_CASE			:bool = True     ,						SCREAMING_SNAKE_CASE			:Optional[str] = None     ,						SCREAMING_SNAKE_CASE			:Optional[str] = ""     ,						**SCREAMING_SNAKE_CASE			:Tuple     ,						)       ->					None:




             '''simple docstring'''
             super().__init__(**SCREAMING_SNAKE_CASE	)
             _a    : List[Any]     					=size if size is not None else {"""height""": 2_2_4, """width""": 2_2_4}
             _a    : Tuple     					=get_size_dict(SCREAMING_SNAKE_CASE	)

             _a    : Dict     					=do_resize
             _a    : Tuple     					=size
             _a    : str     					=resample
             _a    : Dict     					=apply_ocr
             _a    : Union[str, Any]     					=ocr_lang
             _a    : Dict     					=tesseract_config






      def __UpperCAmelCase       ( self			:List[str]     ,						SCREAMING_SNAKE_CASE			:np.ndarray     ,						SCREAMING_SNAKE_CASE			:Dict[str, int]     ,						SCREAMING_SNAKE_CASE			:PILImageResampling = PILImageResampling.BILINEAR     ,						SCREAMING_SNAKE_CASE			:Optional[Union[str, ChannelDimension]] = None     ,						**SCREAMING_SNAKE_CASE			:Dict     ,						)       ->					np.ndarray:




             '''simple docstring'''
             _a    : int     					=get_size_dict(SCREAMING_SNAKE_CASE	)
             if "height" not in size or "width" not in size:
                    raise ValueError(f"The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}"	)
             _a    : Any     					=(size["""height"""], size["""width"""])
             return resize(SCREAMING_SNAKE_CASE     ,						size=SCREAMING_SNAKE_CASE     ,						resample=SCREAMING_SNAKE_CASE     ,						data_format=SCREAMING_SNAKE_CASE     ,						**SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:Dict     ,						SCREAMING_SNAKE_CASE			:ImageInput     ,						SCREAMING_SNAKE_CASE			:bool = None     ,						SCREAMING_SNAKE_CASE			:Dict[str, int] = None     ,						SCREAMING_SNAKE_CASE			:PILImageResampling = None     ,						SCREAMING_SNAKE_CASE			:bool = None     ,						SCREAMING_SNAKE_CASE			:Optional[str] = None     ,						SCREAMING_SNAKE_CASE			:Optional[str] = None     ,						SCREAMING_SNAKE_CASE			:Optional[Union[str, TensorType]] = None     ,						SCREAMING_SNAKE_CASE			:ChannelDimension = ChannelDimension.FIRST     ,						**SCREAMING_SNAKE_CASE			:Optional[Any]     ,						)       ->					PIL.Image.Image:




             '''simple docstring'''
             _a    : Optional[int]     					=do_resize if do_resize is not None else self.do_resize
             _a    : Optional[int]     					=size if size is not None else self.size
             _a    : str     					=get_size_dict(SCREAMING_SNAKE_CASE	)
             _a    : List[str]     					=resample if resample is not None else self.resample
             _a    : int     					=apply_ocr if apply_ocr is not None else self.apply_ocr
             _a    : str     					=ocr_lang if ocr_lang is not None else self.ocr_lang
             _a    : Union[str, Any]     					=tesseract_config if tesseract_config is not None else self.tesseract_config

             _a    : List[str]     					=make_list_of_images(SCREAMING_SNAKE_CASE	)

             if not valid_images(SCREAMING_SNAKE_CASE	):
                    raise ValueError(
                        """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
                        """torch.Tensor, tf.Tensor or jax.ndarray."""	)

             if do_resize and size is None:
                    raise ValueError("""Size must be specified if do_resize is True."""	)

             # All transformations expect numpy arrays.
             _a    : List[Any]     					=[to_numpy_array(SCREAMING_SNAKE_CASE	) for image in images]

             if apply_ocr:
                    requires_backends(self     ,						"""pytesseract"""	)
                    _a    : Any     					=[]
                    _a    : Any     					=[]
                    for image in images:
                           _a					,    _a    : int     					=apply_tesseract(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)
                           words_batch.append(SCREAMING_SNAKE_CASE	)
                           boxes_batch.append(SCREAMING_SNAKE_CASE	)

             if do_resize:
                    _a    : Union[str, Any]     					=[self.resize(image=SCREAMING_SNAKE_CASE     ,						size=SCREAMING_SNAKE_CASE     ,						resample=SCREAMING_SNAKE_CASE	) for image in images]

             # flip color channels from RGB to BGR (as Detectron2 requires this)
             _a    : Dict     					=[flip_channel_order(SCREAMING_SNAKE_CASE	) for image in images]
             _a    : str     					=[to_channel_dimension_format(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	) for image in images]

             _a    : str     					=BatchFeature(data={"""pixel_values""": images}     ,						tensor_type=SCREAMING_SNAKE_CASE	)

             if apply_ocr:
                    _a    : List[Any]     					=words_batch
                    _a    : Dict     					=boxes_batch
             return data


