



'''simple docstring'''

import copy
import inspect
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import VideoMAEConfig
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch
							from torch import nn

							from transformers import (
							    MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
							    VideoMAEForPreTraining,
							    VideoMAEForVideoClassification,
							    VideoMAEModel,
							)
							from transformers.models.videomae.modeling_videomae import VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
							from transformers import VideoMAEImageProcessor

class 	__magic_name__						:







							def __init__(							self					:       str	,_UpperCAmelCase					:       Dict	,_UpperCAmelCase					:       str=13	,_UpperCAmelCase					:       Dict=10	,_UpperCAmelCase					:       str=3	,_UpperCAmelCase					:       Optional[Any]=2	,_UpperCAmelCase					:       Optional[Any]=2	,_UpperCAmelCase					:       List[str]=2	,_UpperCAmelCase					:       Optional[int]=True	,_UpperCAmelCase					:       List[Any]=True	,_UpperCAmelCase					:       Optional[int]=32	,_UpperCAmelCase					:       Dict=5	,_UpperCAmelCase					:       Tuple=4	,_UpperCAmelCase					:       Any=37	,_UpperCAmelCase					:       List[Any]="gelu"	,_UpperCAmelCase					:       List[Any]=0.1	,_UpperCAmelCase					:       Optional[int]=0.1	,_UpperCAmelCase					:       List[Any]=10	,_UpperCAmelCase					:       Any=0.02	,_UpperCAmelCase					:       Any=0.9	,_UpperCAmelCase					:       Optional[int]=None	,):
									_a     :      List[Any]        =							parent
									_a     :      str        =							batch_size
									_a     :      List[Any]        =							image_size
									_a     :      Optional[Any]        =							num_channels
									_a     :      Union[str, Any]        =							patch_size
									_a     :      Dict        =							tubelet_size
									_a     :      Tuple        =							num_frames
									_a     :      List[Any]        =							is_training
									_a     :      Union[str, Any]        =							use_labels
									_a     :      List[Any]        =							hidden_size
									_a     :      str        =							num_hidden_layers
									_a     :      str        =							num_attention_heads
									_a     :      str        =							intermediate_size
									_a     :      Tuple        =							hidden_act
									_a     :      Optional[int]        =							hidden_dropout_prob
									_a     :      int        =							attention_probs_dropout_prob
									_a     :      Optional[int]        =							type_sequence_label_size
									_a     :      List[str]        =							initializer_range
									_a     :      Dict        =							mask_ratio
									_a     :      str        =							scope

									# in VideoMAE, the number of tokens equals num_frames/tubelet_size * num_patches per frame
									_a     :      Tuple        =							(image_size // patch_size) ** 2
									_a     :      int        =							(num_frames // tubelet_size) * self.num_patches_per_frame

									# use this variable to define bool_masked_pos
									_a     :      Optional[int]        =							int(mask_ratio * self.seq_length		)







							def 							__lowercase			(							self					:       List[Any]		):
									_a     :      Union[str, Any]        =							floats_tensor(
									    [self.batch_size, self.num_frames, self.num_channels, self.image_size, self.image_size]		)

									_a     :      Tuple        =							None
									if self.use_labels:
											_a     :      List[str]        =							ids_tensor([self.batch_size]	,self.type_sequence_label_size		)

									_a     :      Optional[int]        =							self.get_config()

									return config, pixel_values, labels







							def 							__lowercase			(							self					:       Union[str, Any]		):
									return VideoMAEConfig(
									    image_size=self.image_size	,patch_size=self.patch_size	,num_channels=self.num_channels	,num_frames=self.num_frames	,tubelet_size=self.tubelet_size	,hidden_size=self.hidden_size	,num_hidden_layers=self.num_hidden_layers	,num_attention_heads=self.num_attention_heads	,intermediate_size=self.intermediate_size	,hidden_act=self.hidden_act	,hidden_dropout_prob=self.hidden_dropout_prob	,attention_probs_dropout_prob=self.attention_probs_dropout_prob	,is_decoder=_UpperCAmelCase	,initializer_range=self.initializer_range	,)







							def 							__lowercase			(							self					:       Tuple	,_UpperCAmelCase					:       List[Any]	,_UpperCAmelCase					:       Optional[int]	,_UpperCAmelCase					:       List[Any]		):
									_a     :      int        =							VideoMAEModel(config=_UpperCAmelCase		)
									model.to(_UpperCAmelCase		)
									model.eval()
									_a     :      Any        =							model(_UpperCAmelCase		)
									self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, self.seq_length, self.hidden_size)		)







							def 							__lowercase			(							self					:       Any	,_UpperCAmelCase					:       str	,_UpperCAmelCase					:       Any	,_UpperCAmelCase					:       str		):
									_a     :      Optional[int]        =							VideoMAEForPreTraining(_UpperCAmelCase		)
									model.to(_UpperCAmelCase		)
									model.eval()
									# important: each video needs to have the same number of masked patches
									# hence we define a single mask, which we then repeat for each example in the batch
									_a     :      Dict        =							torch.ones((self.num_masks,)		)
									_a     :      Union[str, Any]        =							torch.cat([mask, torch.zeros(self.seq_length - mask.size(0		)		)]		)
									_a     :      Tuple        =							mask.expand(self.batch_size	,-1		).bool()

									_a     :      Any        =							model(_UpperCAmelCase	,_UpperCAmelCase		)
									# model only returns predictions for masked patches
									_a     :      Any        =							mask.sum().item()
									_a     :      str        =							3 * self.tubelet_size * self.patch_size**2
									self.parent.assertEqual(result.logits.shape	,(self.batch_size, num_masked_patches, decoder_num_labels)		)







							def 							__lowercase			(							self					:       Union[str, Any]		):
									_a     :      Tuple        =							self.prepare_config_and_inputs()
									_a       ,					_a       ,					_a     :      List[str]        =							config_and_inputs
									_a     :      Any        =							{'pixel_values': pixel_values}
									return config, inputs_dict

@require_torch
class 	__magic_name__						(    _UpperCamelCase			,						_UpperCamelCase			,						unittest.TestCase   ):
							lowerCAmelCase       :    Tuple   				=					(
							    (VideoMAEModel, VideoMAEForPreTraining, VideoMAEForVideoClassification) if is_torch_available() else ()
							)
							lowerCAmelCase       :    List[Any]   				=					(
							    {'feature-extraction': VideoMAEModel, 'video-classification': VideoMAEForVideoClassification}
							    if is_torch_available()
							    else {}
							)

							lowerCAmelCase       :    Optional[int]   				=					False
							lowerCAmelCase       :    Any   				=					False
							lowerCAmelCase       :    List[Any]   				=					False
							lowerCAmelCase       :    Dict   				=					False







							def 							__lowercase			(							self					:       Dict		):
									_a     :      Dict        =							VideoMAEModelTester(self		)
									_a     :      List[str]        =							ConfigTester(self	,config_class=_UpperCAmelCase	,has_text_modality=_UpperCAmelCase	,hidden_size=37		)







							def 							__lowercase			(							self					:       Union[str, Any]	,_UpperCAmelCase					:       Optional[int]	,_UpperCAmelCase					:       Dict	,_UpperCAmelCase					:       Tuple=False		):
									_a     :      Union[str, Any]        =							copy.deepcopy(_UpperCAmelCase		)

									if model_class == VideoMAEForPreTraining:
											# important: each video needs to have the same number of masked patches
											# hence we define a single mask, which we then repeat for each example in the batch
											_a     :      Optional[Any]        =							torch.ones((self.model_tester.num_masks,)		)
											_a     :      str        =							torch.cat([mask, torch.zeros(self.model_tester.seq_length - mask.size(0		)		)]		)
											_a     :      Optional[Any]        =							mask.expand(self.model_tester.batch_size	,-1		).bool()
											_a     :      List[str]        =							bool_masked_pos.to(_UpperCAmelCase		)

									if return_labels:
											if model_class in [
											    *get_values(_UpperCAmelCase		),
											]:
													_a     :      Dict        =							torch.zeros(
													    self.model_tester.batch_size	,dtype=torch.long	,device=_UpperCAmelCase		)

									return inputs_dict







							def 							__lowercase			(							self					:       Optional[Any]		):
									self.config_tester.run_common_tests()







							@unittest.skip(reason='VideoMAE does not use inputs_embeds'		)
							def 							__lowercase			(							self					:       List[str]		):
									pass







							def 							__lowercase			(							self					:       List[str]		):
									_a       ,					_a     :      Dict        =							self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
											_a     :      int        =							model_class(_UpperCAmelCase		)
											self.assertIsInstance(model.get_input_embeddings()	,(nn.Module)		)
											_a     :      Optional[int]        =							model.get_output_embeddings()
											self.assertTrue(x is None or isinstance(_UpperCAmelCase	,nn.Linear		)		)







							def 							__lowercase			(							self					:       str		):
									_a       ,					_a     :      str        =							self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
											_a     :      str        =							model_class(_UpperCAmelCase		)
											_a     :      List[str]        =							inspect.signature(model.forward		)
											# signature.parameters is an OrderedDict => so arg_names order is deterministic
											_a     :      Union[str, Any]        =							[*signature.parameters.keys()]

											_a     :      Optional[Any]        =							['pixel_values']
											self.assertListEqual(arg_names[:1]	,_UpperCAmelCase		)







							def 							__lowercase			(							self					:       int		):
									_a     :      int        =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*_UpperCAmelCase		)







							def 							__lowercase			(							self					:       Any		):
									_a     :      List[str]        =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_pretraining(*_UpperCAmelCase		)







							@slow
							def 							__lowercase			(							self					:       Optional[int]		):
									for model_name in VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_a     :      Dict        =							VideoMAEModel.from_pretrained(_UpperCAmelCase		)
											self.assertIsNotNone(_UpperCAmelCase		)







							def 							__lowercase			(							self					:       Union[str, Any]		):
									if not self.has_attentions:
											pass

									else:
											_a       ,					_a     :      Any        =							self.model_tester.prepare_config_and_inputs_for_common()
											_a     :      List[Any]        =							True

											for model_class in self.all_model_classes:
													_a     :      Dict        =							self.model_tester.seq_length - self.model_tester.num_masks
													_a     :      Optional[int]        =							(
													    num_visible_patches if model_class == VideoMAEForPreTraining else self.model_tester.seq_length
													)

													_a     :      Union[str, Any]        =							True
													_a     :      str        =							False
													_a     :      Optional[int]        =							True
													_a     :      Optional[int]        =							model_class(_UpperCAmelCase		)
													model.to(_UpperCAmelCase		)
													model.eval()
													with torch.no_grad():
															_a     :      Union[str, Any]        =							model(**self._prepare_for_class(_UpperCAmelCase	,_UpperCAmelCase		)		)
													_a     :      Union[str, Any]        =							outputs.attentions
													self.assertEqual(len(_UpperCAmelCase		)	,self.model_tester.num_hidden_layers		)

													# check that output_attentions also work using config
													del inputs_dict["output_attentions"]
													_a     :      Tuple        =							True
													_a     :      Optional[Any]        =							model_class(_UpperCAmelCase		)
													model.to(_UpperCAmelCase		)
													model.eval()
													with torch.no_grad():
															_a     :      Dict        =							model(**self._prepare_for_class(_UpperCAmelCase	,_UpperCAmelCase		)		)
													_a     :      Optional[Any]        =							outputs.attentions
													self.assertEqual(len(_UpperCAmelCase		)	,self.model_tester.num_hidden_layers		)

													self.assertListEqual(
													    list(attentions[0].shape[-3:]		)	,[self.model_tester.num_attention_heads, seq_len, seq_len]	,)
													_a     :      List[str]        =							len(_UpperCAmelCase		)

													# Check attention is always last and order is fine
													_a     :      Dict        =							True
													_a     :      Tuple        =							True
													_a     :      List[Any]        =							model_class(_UpperCAmelCase		)
													model.to(_UpperCAmelCase		)
													model.eval()
													with torch.no_grad():
															_a     :      Dict        =							model(**self._prepare_for_class(_UpperCAmelCase	,_UpperCAmelCase		)		)

													self.assertEqual(out_len + 1	,len(_UpperCAmelCase		)		)

													_a     :      int        =							outputs.attentions

													self.assertEqual(len(_UpperCAmelCase		)	,self.model_tester.num_hidden_layers		)
													self.assertListEqual(
													    list(self_attentions[0].shape[-3:]		)	,[self.model_tester.num_attention_heads, seq_len, seq_len]	,)







							def 							__lowercase			(							self					:       List[Any]		):
									def check_hidden_states_output(_UpperCAmelCase					:       Union[str, Any]	,_UpperCAmelCase					:       List[str]	,_UpperCAmelCase					:       str		):
											_a     :      int        =							model_class(_UpperCAmelCase		)
											model.to(_UpperCAmelCase		)
											model.eval()

											with torch.no_grad():
													_a     :      Any        =							model(**self._prepare_for_class(_UpperCAmelCase	,_UpperCAmelCase		)		)

											_a     :      Union[str, Any]        =							outputs.hidden_states
											_a     :      Tuple        =							self.model_tester.num_hidden_layers + 1
											self.assertEqual(len(_UpperCAmelCase		)	,_UpperCAmelCase		)

											_a     :      Optional[Any]        =							self.model_tester.seq_length - self.model_tester.num_masks
											_a     :      Tuple        =							num_visible_patches if model_class == VideoMAEForPreTraining else self.model_tester.seq_length

											self.assertListEqual(
											    list(hidden_states[0].shape[-2:]		)	,[seq_length, self.model_tester.hidden_size]	,)

									_a       ,					_a     :      str        =							self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
											_a     :      Dict        =							True
											check_hidden_states_output(_UpperCAmelCase	,_UpperCAmelCase	,_UpperCAmelCase		)

											# check that output_hidden_states also work using config
											del inputs_dict["output_hidden_states"]
											_a     :      str        =							True

											check_hidden_states_output(_UpperCAmelCase	,_UpperCAmelCase	,_UpperCAmelCase		)







							@unittest.skip('Will be fixed soon by reducing the size of the model used for common tests.'		)
							def 							__lowercase			(							self					:       Optional[int]		):
									pass



def      __lowerCamelCase     (      )			->      List[Any]:
		_a     :      Optional[int]        =							hf_hub_download(
		    repo_id='hf-internal-testing/spaghetti-video'						,			filename='eating_spaghetti.npy'						,			repo_type='dataset'  )
		_a     :      Dict        =							np.load(lowerCAmelCase_  )
		return list(lowerCAmelCase_  )

@require_torch
@require_vision
class 	__magic_name__						(    unittest.TestCase   ):







							@cached_property
							def 							__lowercase			(							self					:       List[Any]		):
									# logits were tested with a different mean and std, so we use the same here
									return (
									    VideoMAEImageProcessor(image_mean=[0.5, 0.5, 0.5]	,image_std=[0.5, 0.5, 0.5]		)
									    if is_vision_available()
									    else None
									)







							@slow
							def 							__lowercase			(							self					:       int		):
									_a     :      List[str]        =							VideoMAEForVideoClassification.from_pretrained('MCG-NJU/videomae-base-finetuned-kinetics'		).to(
									    _UpperCAmelCase		)

									_a     :      List[str]        =							self.default_image_processor
									_a     :      List[str]        =							prepare_video()
									_a     :      List[str]        =							image_processor(_UpperCAmelCase	,return_tensors='pt'		).to(_UpperCAmelCase		)

									# forward pass
									with torch.no_grad():
											_a     :      List[Any]        =							model(**_UpperCAmelCase		)

									# verify the logits
									_a     :      Optional[int]        =							torch.Size((1, 400)		)
									self.assertEqual(outputs.logits.shape	,_UpperCAmelCase		)

									_a     :      Dict        =							torch.tensor([0.36_69, -0.06_88, -0.24_21]		).to(_UpperCAmelCase		)

									self.assertTrue(torch.allclose(outputs.logits[0, :3]	,_UpperCAmelCase	,atol=1E-4		)		)







							@slow
							def 							__lowercase			(							self					:       Dict		):
									_a     :      int        =							VideoMAEForPreTraining.from_pretrained('MCG-NJU/videomae-base-short'		).to(_UpperCAmelCase		)

									_a     :      int        =							self.default_image_processor
									_a     :      str        =							prepare_video()
									_a     :      str        =							image_processor(_UpperCAmelCase	,return_tensors='pt'		).to(_UpperCAmelCase		)

									# add boolean mask, indicating which patches to mask
									_a     :      Optional[Any]        =							hf_hub_download(repo_id='hf-internal-testing/bool-masked-pos'	,filename='bool_masked_pos.pt'		)
									_a     :      Optional[int]        =							torch.load(_UpperCAmelCase		)

									# forward pass
									with torch.no_grad():
											_a     :      Optional[Any]        =							model(**_UpperCAmelCase		)

									# verify the logits
									_a     :      Any        =							torch.Size([1, 1408, 1536]		)
									_a     :      Optional[Any]        =							torch.tensor(
									    [[0.79_94, 0.96_12, 0.85_08], [0.74_01, 0.89_58, 0.83_02], [0.58_62, 0.74_68, 0.73_25]]	,device=_UpperCAmelCase		)
									self.assertEqual(outputs.logits.shape	,_UpperCAmelCase		)
									self.assertTrue(torch.allclose(outputs.logits[0, :3, :3]	,_UpperCAmelCase	,atol=1E-4		)		)

									# verify the loss (`config.norm_pix_loss` = `True`)
									_a     :      List[Any]        =							torch.tensor([0.51_42]	,device=_UpperCAmelCase		)
									self.assertTrue(torch.allclose(outputs.loss	,_UpperCAmelCase	,atol=1E-4		)		)

									# verify the loss (`config.norm_pix_loss` = `False`)
									_a     :      Union[str, Any]        =							VideoMAEForPreTraining.from_pretrained('MCG-NJU/videomae-base-short'	,norm_pix_loss=_UpperCAmelCase		).to(
									    _UpperCAmelCase		)

									with torch.no_grad():
											_a     :      List[str]        =							model(**_UpperCAmelCase		)

									_a     :      int        =							torch.tensor(torch.tensor([0.64_69]		)	,device=_UpperCAmelCase		)
									self.assertTrue(torch.allclose(outputs.loss	,_UpperCAmelCase	,atol=1E-4		)		)








import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


lowerCAmelCase       =							'<<<<<<< This should probably be modified because it mentions: '

lowerCAmelCase       =							'=======\n>>>>>>>\n'

lowerCAmelCase       =							[
    'TextEncoderConfig',
    'ByteTextEncoder',
    'SubwordTextEncoder',
    'encoder_config',
    'maybe_build_from_corpus',
    'manual_dir',
]

lowerCAmelCase       =							[
    # (pattern, replacement)
    # Order is important here for some replacements
    (R'tfds\.core', R'datasets'),
    (R'tf\.io\.gfile\.GFile', R'open'),
    (R'tf\.([\w\d]+)', R'datasets.Value(\'\1\')'),
    (R'tfds\.features\.Text\(\)', R'datasets.Value(\'string\')'),
    (R'tfds\.features\.Text\(', R'datasets.Value(\'string\'),'),
    (R'features\s*=\s*tfds.features.FeaturesDict\(', R'features=datasets.Features('),
    (R'tfds\.features\.FeaturesDict\(', R'dict('),
    (R'The TensorFlow Datasets Authors', R'The TensorFlow Datasets Authors and the HuggingFace Datasets Authors'),
    (R'tfds\.', R'datasets.'),
    (R'dl_manager\.manual_dir', R'self.config.data_dir'),
    (R'self\.builder_config', R'self.config'),
]
def 			_a (      SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		return ConvertCommand(args.tfds_path						,				args.datasets_directory     )


class        _a							(			UpperCamelCase__			):

							@staticmethod
							def       lowerCamelCase_ (							UpperCamelCase_:	ArgumentParser							)			->   int:





									"""simple docstring"""







									lowercase__            =      parser.add_parser(
									    '''convert'''    ,						help='''Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.'''    ,						)
									train_parser.add_argument(
									    '''--tfds_path'''    ,						type=UpperCamelCase_    ,						required=UpperCamelCase_    ,						help='''Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.'''    ,						)
									train_parser.add_argument(
									    '''--datasets_directory'''    ,						type=UpperCamelCase_    ,						required=UpperCamelCase_    ,						help='''Path to the HuggingFace Datasets folder.'''							)
									train_parser.set_defaults(func=UpperCamelCase_							)

							def __init__(							self:	int    ,						UpperCamelCase_:	str    ,						UpperCamelCase_:	str    ,						*UpperCamelCase_:	Optional[int]							)			->   Union[str, Any]:





									"""simple docstring"""







									lowercase__            =      get_logger('''datasets-cli/converting'''							)

									lowercase__            =      tfds_path
									lowercase__            =      datasets_directory


							def       lowerCamelCase_ (							self:	int							)			->   Optional[int]:





									"""simple docstring"""







									if os.path.isdir(self._tfds_path							):
											lowercase__            =      os.path.abspath(self._tfds_path							)
									elif os.path.isfile(self._tfds_path							):
											lowercase__            =      os.path.dirname(self._tfds_path							)
									else:
											raise ValueError('''--tfds_path is neither a directory nor a file. Please check path.'''							)

									lowercase__            =      os.path.abspath(self._datasets_directory							)

									self._logger.info(f'Converting datasets from {abs_tfds_path} to {abs_datasets_path}'							)

									lowercase__            =      []
									lowercase__            =      []
									lowercase__            =      {}

									if os.path.isdir(self._tfds_path							):
											lowercase__            =      os.listdir(UpperCamelCase_							)
									else:
											lowercase__            =      [os.path.basename(self._tfds_path							)]

									for f_name in file_names:
											self._logger.info(f'Looking at file {f_name}'							)
											lowercase__            =      os.path.join(UpperCamelCase_    ,						UpperCamelCase_							)
											lowercase__            =      os.path.join(UpperCamelCase_    ,						UpperCamelCase_							)

											if not os.path.isfile(UpperCamelCase_							) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
													self._logger.info('''Skipping file'''							)
													continue

											with open(UpperCamelCase_    ,						encoding='''utf-8'''							) as f:
													lowercase__            =      f.readlines()

											lowercase__            =      []
											lowercase__            =      False
											lowercase__            =      False
											lowercase__            =      []
											for line in lines:
													lowercase__            =      line

													# Convert imports
													if "import tensorflow.compat.v2 as tf" in out_line:
															continue
													elif "@tfds.core" in out_line:
															continue
													elif "builder=self" in out_line:
															continue
													elif "import tensorflow_datasets.public_api as tfds" in out_line:
															lowercase__            =      '''import datasets\n'''
													elif "import tensorflow" in out_line:
															# order is important here
															lowercase__            =      ''''''
															continue
													elif "from absl import logging" in out_line:
															lowercase__            =      '''from datasets import logging\n'''
													elif "getLogger" in out_line:
															lowercase__            =      out_line.replace('''getLogger'''    ,						'''get_logger'''							)
													elif any(expression in out_line for expression in TO_HIGHLIGHT							):
															lowercase__            =      True
															lowercase__            =      list(filter(lambda UpperCamelCase_							: e in out_line    ,						UpperCamelCase_							)							)
															out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(UpperCamelCase_							) + '''\n'''							)
															out_lines.append(UpperCamelCase_							)
															out_lines.append(UpperCamelCase_							)
															continue
													else:
															for pattern, replacement in TO_CONVERT:
																	lowercase__            =      re.sub(UpperCamelCase_    ,						UpperCamelCase_    ,						UpperCamelCase_							)

                # Take care of saving utilities (to later move them together with main script)
													if "tensorflow_datasets" in out_line:
															lowercase__            =      re.match(r'''from\stensorflow_datasets.*import\s([^\.\r\n]+)'''    ,						UpperCamelCase_							)
															tfds_imports.extend(imp.strip() for imp in match.group(1							).split(''','''							)							)
															lowercase__            =      '''from . import ''' + match.group(1							)

													# Check we have not forget anything
													if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
															raise ValueError(f'Error converting {out_line.strip()}'							)

													if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
															lowercase__            =      True
													out_lines.append(UpperCamelCase_							)

											if is_builder or "wmt" in f_name:
													# We create a new directory for each dataset
													lowercase__            =      f_name.replace('''.py'''    ,						''''''							)
													lowercase__            =      os.path.join(UpperCamelCase_    ,						UpperCamelCase_							)
													lowercase__            =      os.path.join(UpperCamelCase_    ,						UpperCamelCase_							)
													os.makedirs(UpperCamelCase_    ,						exist_ok=UpperCamelCase_							)
													self._logger.info(f'Adding directory {output_dir}'							)
													imports_to_builder_map.update({imp: output_dir for imp in tfds_imports}							)
											else:
													# Utilities will be moved at the end
													utils_files.append(UpperCamelCase_							)

											if needs_manual_update:
													with_manual_update.append(UpperCamelCase_							)

											with open(UpperCamelCase_    ,						'''w'''    ,						encoding='''utf-8'''							) as f:
													f.writelines(UpperCamelCase_							)
											self._logger.info(f'Converted in {output_file}'							)

									for utils_file in utils_files:
											try:
													lowercase__            =      os.path.basename(UpperCamelCase_							)
													lowercase__            =      imports_to_builder_map[f_name.replace('''.py'''    ,						''''''							)]
													self._logger.info(f'Moving {dest_folder} to {utils_file}'							)
													shutil.copy(UpperCamelCase_    ,						UpperCamelCase_							)
											except KeyError:
													self._logger.error(f'Cannot find destination folder for {utils_file}. Please copy manually.'							)

									if with_manual_update:
											for file_path in with_manual_update:
													self._logger.warning(
													    f'You need to manually update file {file_path} to remove configurations using \'TextEncoderConfig\'.'							)


