



'''simple docstring'''
import tempfile
import unittest

import numpy as np
from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_flax


if is_flax_available():
					import os

					from flax.core.frozen_dict import unfreeze
					from flax.traverse_util import flatten_dict

					from transformers import FlaxBertModel

					snake_case_ :	Dict         =  '0.12'  # assumed parallelism: 8






@require_flax
@is_staging_test
class 							lowercase__	(							unittest.TestCase   ):





	@classmethod
	def 	UpperCamelCase_  ( cls					:				str ):







							'''simple docstring'''
							_UpperCamelCase					:		List[str] 				=   TOKEN
							HfFolder.save_token(lowerCamelCase__ )





	@classmethod
	def 	UpperCamelCase_  ( cls					:				List[Any] ):







							'''simple docstring'''
							try:
													delete_repo(token=cls._token ,repo_id='test-model-flax' )
							except HTTPError:
													pass

							try:
													delete_repo(token=cls._token ,repo_id='valid_org/test-model-flax-org' )
							except HTTPError:
													pass





	def 	UpperCamelCase_  ( self					:				Tuple ):







							'''simple docstring'''
							_UpperCamelCase					:		Tuple 				=   BertConfig(
							    vocab_size=99 ,hidden_size=32 ,num_hidden_layers=5 ,num_attention_heads=4 ,intermediate_size=37 )
							_UpperCamelCase					:		int 				=   FlaxBertModel(lowerCamelCase__ )
							model.push_to_hub('test-model-flax' ,use_auth_token=self._token )

							_UpperCamelCase					:		Union[str, Any] 				=   FlaxBertModel.from_pretrained(F'{USER}/test-model-flax' )

							_UpperCamelCase					:		Optional[int] 				=   flatten_dict(unfreeze(model.params ) )
							_UpperCamelCase					:		List[Any] 				=   flatten_dict(unfreeze(new_model.params ) )

							for key in base_params.keys():
													_UpperCamelCase					:		Union[str, Any] 				=   (base_params[key] - new_params[key]).sum().item()
													self.assertLessEqual(lowerCamelCase__ ,1E-3 ,msg=F'{key} not identical' )

							# Reset repo
							delete_repo(token=self._token ,repo_id='test-model-flax' )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
													model.save_pretrained(lowerCamelCase__ ,repo_id='test-model-flax' ,push_to_hub=lowerCamelCase__ ,use_auth_token=self._token )

							_UpperCamelCase					:		List[str] 				=   FlaxBertModel.from_pretrained(F'{USER}/test-model-flax' )

							_UpperCamelCase					:		Any 				=   flatten_dict(unfreeze(model.params ) )
							_UpperCamelCase					:		Tuple 				=   flatten_dict(unfreeze(new_model.params ) )

							for key in base_params.keys():
													_UpperCamelCase					:		str 				=   (base_params[key] - new_params[key]).sum().item()
													self.assertLessEqual(lowerCamelCase__ ,1E-3 ,msg=F'{key} not identical' )





	def 	UpperCamelCase_  ( self					:				str ):







							'''simple docstring'''
							_UpperCamelCase					:		Dict 				=   BertConfig(
							    vocab_size=99 ,hidden_size=32 ,num_hidden_layers=5 ,num_attention_heads=4 ,intermediate_size=37 )
							_UpperCamelCase					:		List[Any] 				=   FlaxBertModel(lowerCamelCase__ )
							model.push_to_hub('valid_org/test-model-flax-org' ,use_auth_token=self._token )

							_UpperCamelCase					:		List[str] 				=   FlaxBertModel.from_pretrained('valid_org/test-model-flax-org' )

							_UpperCamelCase					:		Any 				=   flatten_dict(unfreeze(model.params ) )
							_UpperCamelCase					:		str 				=   flatten_dict(unfreeze(new_model.params ) )

							for key in base_params.keys():
													_UpperCamelCase					:		Tuple 				=   (base_params[key] - new_params[key]).sum().item()
													self.assertLessEqual(lowerCamelCase__ ,1E-3 ,msg=F'{key} not identical' )

							# Reset repo
							delete_repo(token=self._token ,repo_id='valid_org/test-model-flax-org' )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
													model.save_pretrained(
													    lowerCamelCase__ ,repo_id='valid_org/test-model-flax-org' ,push_to_hub=lowerCamelCase__ ,use_auth_token=self._token )

							_UpperCamelCase					:		str 				=   FlaxBertModel.from_pretrained('valid_org/test-model-flax-org' )

							_UpperCamelCase					:		Tuple 				=   flatten_dict(unfreeze(model.params ) )
							_UpperCamelCase					:		Dict 				=   flatten_dict(unfreeze(new_model.params ) )

							for key in base_params.keys():
													_UpperCamelCase					:		Tuple 				=   (base_params[key] - new_params[key]).sum().item()
													self.assertLessEqual(lowerCamelCase__ ,1E-3 ,msg=F'{key} not identical' )






def A__    (     UpperCAmelCase_  ,  UpperCAmelCase_							):
						_UpperCamelCase					:		Union[str, Any] 				=   True
						_UpperCamelCase					:		Union[str, Any] 				=   flatten_dict(modela.params							)
						_UpperCamelCase					:		Any 				=   flatten_dict(modela.params							)
						for key in flat_params_a.keys():
												if np.sum(np.abs(flat_params_a[key] - flat_params_a[key]							)							) > 1E-4:
																		_UpperCamelCase					:		List[str] 				=   False

						return models_are_equal






@require_flax
class 							lowercase__	(							unittest.TestCase   ):





	def 	UpperCamelCase_  ( self					:				int ):







							'''simple docstring'''
							_UpperCamelCase					:		Dict 				=   BertConfig.from_pretrained('hf-internal-testing/tiny-bert-flax-only' )
							_UpperCamelCase					:		Optional[Any] 				=   FlaxBertModel(lowerCamelCase__ )

							_UpperCamelCase					:		Union[str, Any] 				=   'bert'
							with tempfile.TemporaryDirectory() as tmp_dir:
													model.save_pretrained(os.path.join(lowerCamelCase__ ,lowerCamelCase__ ) )

													with self.assertRaises(lowerCamelCase__ ):
																			_UpperCamelCase					:		Tuple 				=   FlaxBertModel.from_pretrained(lowerCamelCase__ )

													_UpperCamelCase					:		Tuple 				=   FlaxBertModel.from_pretrained(lowerCamelCase__ ,subfolder=lowerCamelCase__ )

							self.assertTrue(check_models_equal(lowerCamelCase__ ,lowerCamelCase__ ) )





	def 	UpperCamelCase_  ( self					:				Any ):







							'''simple docstring'''
							_UpperCamelCase					:		List[str] 				=   BertConfig.from_pretrained('hf-internal-testing/tiny-bert-flax-only' )
							_UpperCamelCase					:		str 				=   FlaxBertModel(lowerCamelCase__ )

							_UpperCamelCase					:		Any 				=   'bert'
							with tempfile.TemporaryDirectory() as tmp_dir:
													model.save_pretrained(os.path.join(lowerCamelCase__ ,lowerCamelCase__ ) ,max_shard_size='10KB' )

													with self.assertRaises(lowerCamelCase__ ):
																			_UpperCamelCase					:		Tuple 				=   FlaxBertModel.from_pretrained(lowerCamelCase__ )

													_UpperCamelCase					:		Tuple 				=   FlaxBertModel.from_pretrained(lowerCamelCase__ ,subfolder=lowerCamelCase__ )

							self.assertTrue(check_models_equal(lowerCamelCase__ ,lowerCamelCase__ ) )





	def 	UpperCamelCase_  ( self					:				Optional[Any] ):







							'''simple docstring'''
							_UpperCamelCase					:		List[str] 				=   'bert'
							_UpperCamelCase					:		Tuple 				=   'hf-internal-testing/tiny-random-bert-subfolder'

							with self.assertRaises(lowerCamelCase__ ):
													_UpperCamelCase					:		Tuple 				=   FlaxBertModel.from_pretrained(lowerCamelCase__ )

							_UpperCamelCase					:		Optional[int] 				=   FlaxBertModel.from_pretrained(lowerCamelCase__ ,subfolder=lowerCamelCase__ )

							self.assertIsNotNone(lowerCamelCase__ )





	def 	UpperCamelCase_  ( self					:				Union[str, Any] ):







							'''simple docstring'''
							_UpperCamelCase					:		Any 				=   'bert'
							_UpperCamelCase					:		Union[str, Any] 				=   'hf-internal-testing/tiny-random-bert-sharded-subfolder'
							with self.assertRaises(lowerCamelCase__ ):
													_UpperCamelCase					:		Dict 				=   FlaxBertModel.from_pretrained(lowerCamelCase__ )

							_UpperCamelCase					:		Dict 				=   FlaxBertModel.from_pretrained(lowerCamelCase__ ,subfolder=lowerCamelCase__ )

							self.assertIsNotNone(lowerCamelCase__ )






'''simple docstring'''







import contextlib
import os
import sqlitea

import pytest

from datasets import Dataset, Features, Value
from datasets.io.sql import SqlDatasetReader, SqlDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases, require_sqlalchemy


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Any    ,_UpperCAmelCase	:	str  )  ->			Dict:
       assert isinstance(_UpperCAmelCase    ,_UpperCAmelCase  )
       assert dataset.num_rows == 4
       assert dataset.num_columns == 3
       assert dataset.column_names == ["col_1", "col_2", "col_3"]
       for feature, expected_dtype in expected_features.items():
              assert dataset.features[feature].dtype == expected_dtype


@require_sqlalchemy
@pytest.mark.parametrize("""keep_in_memory"""    ,[False, True]  )
def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Optional[Any]    ,_UpperCAmelCase	:	Optional[int]    ,_UpperCAmelCase	:	Optional[int]    ,_UpperCAmelCase	:	str  )  ->			Optional[Any]:
       _a    : Any     					=tmp_path / """cache"""
       _a    : int     					={"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
       with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
              _a    : Tuple     					=SqlDatasetReader(
                  """dataset"""    ,"""sqlite:///""" + sqlite_path    ,cache_dir=_UpperCAmelCase    ,keep_in_memory=_UpperCAmelCase  ).read()
       _check_sql_dataset(_UpperCAmelCase    ,_UpperCAmelCase  )


@require_sqlalchemy
@pytest.mark.parametrize(
    """features"""    ,[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]    ,)
def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	List[Any]    ,_UpperCAmelCase	:	Dict    ,_UpperCAmelCase	:	Optional[Any]    ,_UpperCAmelCase	:	int  )  ->			List[Any]:
       _a    : Union[str, Any]     					=tmp_path / """cache"""
       _a    : str     					={"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
       _a    : Optional[int]     					=features.copy() if features else default_expected_features
       _a    : Union[str, Any]     					=(
           Features({feature: Value(_UpperCAmelCase  ) for feature, dtype in features.items()}  ) if features is not None else None
       )
       _a    : Optional[Any]     					=SqlDatasetReader("""dataset"""    ,"""sqlite:///""" + sqlite_path    ,features=_UpperCAmelCase    ,cache_dir=_UpperCAmelCase  ).read()
       _check_sql_dataset(_UpperCAmelCase    ,_UpperCAmelCase  )


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Optional[Any]  )  ->			List[str]:
       with contextlib.closing(sqlitea.connect(_UpperCAmelCase  )  ) as con:
              _a    : Any     					=con.cursor()
              cur.execute("""SELECT * FROM dataset"""  )
              for row in cur:
                     yield row


@require_sqlalchemy
def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Dict    ,_UpperCAmelCase	:	Tuple    ,_UpperCAmelCase	:	List[str]  )  ->			Union[str, Any]:
       _a    : Union[str, Any]     					=tmp_path / """cache"""
       _a    : Union[str, Any]     					=os.path.join(_UpperCAmelCase    ,"""tmp.sql"""  )
       _a    : Tuple     					=SqlDatasetReader("""dataset"""    ,"""sqlite:///""" + sqlite_path    ,cache_dir=_UpperCAmelCase  ).read()
       SqlDatasetWriter(_UpperCAmelCase    ,"""dataset"""    ,"""sqlite:///""" + output_sqlite_path    ,num_proc=1  ).write()

       _a    : Tuple     					=iter_sql_file(_UpperCAmelCase  )
       _a    : List[Any]     					=iter_sql_file(_UpperCAmelCase  )

       for rowa, rowa in zip(_UpperCAmelCase    ,_UpperCAmelCase  ):
              assert rowa == rowa


@require_sqlalchemy
def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Optional[int]    ,_UpperCAmelCase	:	Any    ,_UpperCAmelCase	:	List[Any]  )  ->			Optional[int]:
       _a    : int     					=tmp_path / """cache"""
       _a    : Any     					=os.path.join(_UpperCAmelCase    ,"""tmp.sql"""  )
       _a    : Union[str, Any]     					=SqlDatasetReader("""dataset"""    ,"""sqlite:///""" + sqlite_path    ,cache_dir=_UpperCAmelCase  ).read()
       SqlDatasetWriter(_UpperCAmelCase    ,"""dataset"""    ,"""sqlite:///""" + output_sqlite_path    ,num_proc=2  ).write()

       _a    : List[Any]     					=iter_sql_file(_UpperCAmelCase  )
       _a    : str     					=iter_sql_file(_UpperCAmelCase  )

       for rowa, rowa in zip(_UpperCAmelCase    ,_UpperCAmelCase  ):
              assert rowa == rowa


@require_sqlalchemy
def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Union[str, Any]    ,_UpperCAmelCase	:	str    ,_UpperCAmelCase	:	List[Any]  )  ->			List[str]:
       _a    : List[str]     					=tmp_path / """cache"""
       _a    : Dict     					=os.path.join(_UpperCAmelCase    ,"""tmp.sql"""  )
       _a    : Optional[Any]     					=SqlDatasetReader("""dataset"""    ,"""sqlite:///""" + sqlite_path    ,cache_dir=_UpperCAmelCase  ).read()
       with pytest.raises(_UpperCAmelCase  ):
              SqlDatasetWriter(_UpperCAmelCase    ,"""dataset"""    ,"""sqlite:///""" + output_sqlite_path    ,num_proc=0  ).write()


