



"""simple docstring"""
import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam
class 							_UpperCAmelCase							( datasets.BeamBasedBuilder      ):





     '''simple docstring'''



     def  __lowerCAmelCase	(					self		)							->					Optional[int]:
         return datasets.DatasetInfo(
             features=datasets.Features({'''content''': datasets.Value('''string'''		)}		)       ,	supervised_keys=A       ,	)





     def  __lowerCAmelCase	(					self       ,	A       ,	A		)							->					str:
         return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,	gen_kwargs={'''examples''': get_test_dummy_examples()}		)]





     def  __lowerCAmelCase	(					self       ,	A       ,	A		)							->					Optional[Any]:
         import apache_beam as beam

         return pipeline | "Load Examples" >> beam.Create(A		)



class 							_UpperCAmelCase							( datasets.BeamBasedBuilder      ):





     '''simple docstring'''



     def  __lowerCAmelCase	(					self		)							->					Dict:
         return datasets.DatasetInfo(
             features=datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''		)}		)}		)       ,	supervised_keys=A       ,	)





     def  __lowerCAmelCase	(					self       ,	A       ,	A		)							->					Optional[int]:
         return [
             datasets.SplitGenerator(name=datasets.Split.TRAIN       ,	gen_kwargs={'''examples''': get_test_nested_examples()}		)
         ]





     def  __lowerCAmelCase	(					self       ,	A       ,	A		)							->					str:
         import apache_beam as beam

         return pipeline | "Load Examples" >> beam.Create(A		)





def  lowerCamelCase_						():
    return [(i, {"content": content}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']							)]





def  lowerCamelCase_						():
    return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']							)]


class 							_UpperCAmelCase							( a      ):





     '''simple docstring'''



     @require_beam
     def  __lowerCAmelCase	(					self		)							->					Union[str, Any]:
         _UpperCAmelCase							:  Optional[int]											=    len(get_test_dummy_examples()		)
         with tempfile.TemporaryDirectory() as tmp_cache_dir:
             _UpperCAmelCase							:  str											=    DummyBeamDataset(cache_dir=A       ,	beam_runner='''DirectRunner'''		)
             builder.download_and_prepare()
             self.assertTrue(
                 os.path.exists(
                     os.path.join(A       ,	builder.name       ,	'''default'''       ,	'''0.0.0'''       ,	f'{builder.name}-train.arrow'		)		)		)
             self.assertDictEqual(builder.info.features       ,	datasets.Features({'''content''': datasets.Value('''string'''		)}		)		)
             _UpperCAmelCase							:  Optional[Any]											=    builder.as_dataset()
             self.assertEqual(dset['''train'''].num_rows       ,	A		)
             self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples       ,	A		)
             self.assertDictEqual(dset['''train'''][0]       ,	get_test_dummy_examples()[0][1]		)
             self.assertDictEqual(
                 dset['''train'''][expected_num_examples - 1]       ,	get_test_dummy_examples()[expected_num_examples - 1][1]		)
             self.assertTrue(
                 os.path.exists(os.path.join(A       ,	builder.name       ,	'''default'''       ,	'''0.0.0'''       ,	'''dataset_info.json'''		)		)		)
             del dset





     @require_beam
     def  __lowerCAmelCase	(					self		)							->					Optional[int]:
         import apache_beam as beam

         _UpperCAmelCase							:  Any											=    beam.io.parquetio.WriteToParquet

         _UpperCAmelCase							:  Optional[int]											=    len(get_test_dummy_examples()		)
         with tempfile.TemporaryDirectory() as tmp_cache_dir:
             _UpperCAmelCase							:  Optional[Any]											=    DummyBeamDataset(cache_dir=A       ,	beam_runner='''DirectRunner'''		)
             with patch('''apache_beam.io.parquetio.WriteToParquet'''		) as write_parquet_mock:
                 _UpperCAmelCase							:  Union[str, Any]											=    partial(A       ,	num_shards=2		)
                 builder.download_and_prepare()
             self.assertTrue(
                 os.path.exists(
                     os.path.join(
                         A       ,	builder.name       ,	'''default'''       ,	'''0.0.0'''       ,	f'{builder.name}-train-00000-of-00002.arrow'		)		)		)
             self.assertTrue(
                 os.path.exists(
                     os.path.join(
                         A       ,	builder.name       ,	'''default'''       ,	'''0.0.0'''       ,	f'{builder.name}-train-00000-of-00002.arrow'		)		)		)
             self.assertDictEqual(builder.info.features       ,	datasets.Features({'''content''': datasets.Value('''string'''		)}		)		)
             _UpperCAmelCase							:  Tuple											=    builder.as_dataset()
             self.assertEqual(dset['''train'''].num_rows       ,	A		)
             self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples       ,	A		)
             # Order is not preserved when sharding, so we just check that all the elements are there
             self.assertListEqual(sorted(dset['''train''']['''content''']		)       ,	sorted(['''foo''', '''bar''', '''foobar''']		)		)
             self.assertTrue(
                 os.path.exists(os.path.join(A       ,	builder.name       ,	'''default'''       ,	'''0.0.0'''       ,	'''dataset_info.json'''		)		)		)
             del dset





     @require_beam
     def  __lowerCAmelCase	(					self		)							->					str:
         with tempfile.TemporaryDirectory() as tmp_cache_dir:
             _UpperCAmelCase							:  List[str]											=    DummyBeamDataset(cache_dir=A		)
             self.assertRaises(datasets.builder.MissingBeamOptions       ,	builder.download_and_prepare		)





     @require_beam
     def  __lowerCAmelCase	(					self		)							->					List[str]:
         _UpperCAmelCase							:  List[Any]											=    len(get_test_nested_examples()		)
         with tempfile.TemporaryDirectory() as tmp_cache_dir:
             _UpperCAmelCase							:  Optional[int]											=    NestedBeamDataset(cache_dir=A       ,	beam_runner='''DirectRunner'''		)
             builder.download_and_prepare()
             self.assertTrue(
                 os.path.exists(
                     os.path.join(A       ,	builder.name       ,	'''default'''       ,	'''0.0.0'''       ,	f'{builder.name}-train.arrow'		)		)		)
             self.assertDictEqual(
                 builder.info.features       ,	datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''		)}		)}		)		)
             _UpperCAmelCase							:  str											=    builder.as_dataset()
             self.assertEqual(dset['''train'''].num_rows       ,	A		)
             self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples       ,	A		)
             self.assertDictEqual(dset['''train'''][0]       ,	get_test_nested_examples()[0][1]		)
             self.assertDictEqual(
                 dset['''train'''][expected_num_examples - 1]       ,	get_test_nested_examples()[expected_num_examples - 1][1]		)
             self.assertTrue(
                 os.path.exists(os.path.join(A       ,	builder.name       ,	'''default'''       ,	'''0.0.0'''       ,	'''dataset_info.json'''		)		)		)
             del dset







"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCAmelCase    :int     =					logging.get_logger(__name__)

_lowerCAmelCase    :Union[str, Any]     =					{
    'alibaba-damo/mgp-str-base': 'https://huggingface.co/alibaba-damo/mgp-str-base/resolve/main/config.json',
}
class 							_UpperCAmelCase							( a      ):





     '''simple docstring'''



     a__									='''mgp-str'''





     def __init__(					self       ,	A=[3_2, 1_2_8]       ,	A=4       ,	A=3       ,	A=2_7       ,	A=3_8       ,	A=5_0_2_5_7       ,	A=3_0_5_2_2       ,	A=7_6_8       ,	A=1_2       ,	A=1_2       ,	A=4.0       ,	A=True       ,	A=False       ,	A=1E-5       ,	A=0.0       ,	A=0.0       ,	A=0.0       ,	A=False       ,	A=0.02       ,	**A       ,	)							->					Union[str, Any]:
         super().__init__(**A		)

         _UpperCAmelCase							:  Any											=    image_size
         _UpperCAmelCase							:  str											=    patch_size
         _UpperCAmelCase							:  Dict											=    num_channels
         _UpperCAmelCase							:  Dict											=    max_token_length
         _UpperCAmelCase							:  Optional[Any]											=    num_character_labels
         _UpperCAmelCase							:  int											=    num_bpe_labels
         _UpperCAmelCase							:  List[str]											=    num_wordpiece_labels
         _UpperCAmelCase							:  Optional[int]											=    hidden_size
         _UpperCAmelCase							:  Any											=    num_hidden_layers
         _UpperCAmelCase							:  List[Any]											=    num_attention_heads
         _UpperCAmelCase							:  List[Any]											=    mlp_ratio
         _UpperCAmelCase							:  List[str]											=    distilled
         _UpperCAmelCase							:  Optional[int]											=    layer_norm_eps
         _UpperCAmelCase							:  str											=    drop_rate
         _UpperCAmelCase							:  List[Any]											=    qkv_bias
         _UpperCAmelCase							:  List[str]											=    attn_drop_rate
         _UpperCAmelCase							:  Dict											=    drop_path_rate
         _UpperCAmelCase							:  Union[str, Any]											=    output_aa_attentions
         _UpperCAmelCase							:  List[str]											=    initializer_range


