


'''simple docstring'''
from typing import List, Optional, Union

import torch
from transformers import (
    XLMRobertaTokenizer,
)

from ...models import UNetaDConditionModel, VQModel
from ...pipelines import DiffusionPipeline
from ...pipelines.pipeline_utils import ImagePipelineOutput
from ...schedulers import DDIMScheduler, DDPMScheduler
from ...utils import (
    is_accelerate_available,
    is_accelerate_version,
    logging,
    randn_tensor,
    replace_example_docstring,
)
from .text_encoder import MultilingualCLIP


__snake_case							      =     logging.get_logger(__name__)  # pylint: disable=invalid-name

__snake_case							      =     '''
    Examples:
        ```py
        >>> from diffusers import KandinskyPipeline, KandinskyPriorPipeline
        >>> import torch

        >>> pipe_prior = KandinskyPriorPipeline.from_pretrained("kandinsky-community/Kandinsky-2-1-prior")
        >>> pipe_prior.to("cuda")

        >>> prompt = "red cat, 4k photo"
        >>> out = pipe_prior(prompt)
        >>> image_emb = out.image_embeds
        >>> negative_image_emb = out.negative_image_embeds

        >>> pipe = KandinskyPipeline.from_pretrained("kandinsky-community/kandinsky-2-1")
        >>> pipe.to("cuda")

        >>> image = pipe(
        ...     prompt,
        ...     image_embeds=image_emb,
        ...     negative_image_embeds=negative_image_emb,
        ...     height=768,
        ...     width=768,
        ...     num_inference_steps=100,
        ... ).images

        >>> image[0].save("cat.png")
        ```
'''


def 			a      (			__a ,					__a ,					__a=8   )  ->	Union[str, Any]:



					'''simple docstring'''






					UpperCamelCase__   :Any												= h // scale_factor**2
					if h % scale_factor**2 != 0:
										new_h += 1
					UpperCamelCase__   :Tuple												= w // scale_factor**2
					if w % scale_factor**2 != 0:
										new_w += 1
					return new_h * scale_factor, new_w * scale_factor







class    lowercase	(							A__					):

		"""simple docstring"""

		def __init__(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   ):

							'''simple docstring'''


							super().__init__()

							self.register_modules(
							    text_encoder=UpperCamelCase_    ,   tokenizer=UpperCamelCase_    ,   unet=UpperCamelCase_    ,   scheduler=UpperCamelCase_    ,   movq=UpperCamelCase_    ,   )
							UpperCamelCase__   :Union[str, Any]												= 2 ** (len(self.movq.config.block_out_channels							) - 1)




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_							):

							'''simple docstring'''


							if latents is None:
												UpperCamelCase__   :str												= randn_tensor(UpperCamelCase_    ,   generator=UpperCamelCase_    ,   device=UpperCamelCase_    ,   dtype=UpperCamelCase_							)
							else:
												if latents.shape != shape:
																	raise ValueError(F'''Unexpected latents shape, got {latents.shape}, expected {shape}'''							)
												UpperCamelCase__   :Dict												= latents.to(UpperCamelCase_							)

							UpperCamelCase__   :Any												= latents * scheduler.init_noise_sigma
							return latents




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_=None    ,   ):

							'''simple docstring'''


							UpperCamelCase__   :List[Any]												= len(UpperCamelCase_							) if isinstance(UpperCamelCase_    ,   UpperCamelCase_							) else 1
							# get prompt text embeddings
							UpperCamelCase__   :List[str]												= self.tokenizer(
							    UpperCamelCase_    ,   padding='''max_length'''    ,   truncation=UpperCamelCase_    ,   max_length=77    ,   return_attention_mask=UpperCamelCase_    ,   add_special_tokens=UpperCamelCase_    ,   return_tensors='''pt'''    ,   )

							UpperCamelCase__   :str												= text_inputs.input_ids
							UpperCamelCase__   :List[str]												= self.tokenizer(UpperCamelCase_    ,   padding='''longest'''    ,   return_tensors='''pt'''							).input_ids

							if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(UpperCamelCase_    ,   UpperCamelCase_							):
												UpperCamelCase__   :Tuple												= self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]							)
												logger.warning(
												    '''The following part of your input was truncated because CLIP can only handle sequences up to'''
												    F''' {self.tokenizer.model_max_length} tokens: {removed_text}'''							)

							UpperCamelCase__   :Union[str, Any]												= text_input_ids.to(UpperCamelCase_							)
							UpperCamelCase__   :Optional[int]												= text_inputs.attention_mask.to(UpperCamelCase_							)

							UpperCamelCase__					,				UpperCamelCase__   :Optional[int]												= self.text_encoder(
							    input_ids=UpperCamelCase_    ,   attention_mask=UpperCamelCase_							)

							UpperCamelCase__   :Tuple												= prompt_embeds.repeat_interleave(UpperCamelCase_    ,   dim=0							)
							UpperCamelCase__   :List[Any]												= text_encoder_hidden_states.repeat_interleave(UpperCamelCase_    ,   dim=0							)
							UpperCamelCase__   :str												= text_mask.repeat_interleave(UpperCamelCase_    ,   dim=0							)

							if do_classifier_free_guidance:
												UpperCamelCase__   :List[str]
												if negative_prompt is None:
																	UpperCamelCase__   :Dict												= [''''''] * batch_size
												elif type(UpperCamelCase_							) is not type(UpperCamelCase_							):
																	raise TypeError(
																	    F'''`negative_prompt` should be the same type to `prompt`, but got {type(UpperCamelCase_							)} !='''
																	    F''' {type(UpperCamelCase_							)}.'''							)
												elif isinstance(UpperCamelCase_    ,   UpperCamelCase_							):
																	UpperCamelCase__   :List[str]												= [negative_prompt]
												elif batch_size != len(UpperCamelCase_							):
																	raise ValueError(
																	    F'''`negative_prompt`: {negative_prompt} has batch size {len(UpperCamelCase_							)}, but `prompt`:'''
																	    F''' {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches'''
																	    ''' the batch size of `prompt`.'''							)
												else:
																	UpperCamelCase__   :Tuple												= negative_prompt

												UpperCamelCase__   :int												= self.tokenizer(
												    UpperCamelCase_    ,   padding='''max_length'''    ,   max_length=77    ,   truncation=UpperCamelCase_    ,   return_attention_mask=UpperCamelCase_    ,   add_special_tokens=UpperCamelCase_    ,   return_tensors='''pt'''    ,   )
												UpperCamelCase__   :List[str]												= uncond_input.input_ids.to(UpperCamelCase_							)
												UpperCamelCase__   :Union[str, Any]												= uncond_input.attention_mask.to(UpperCamelCase_							)

												UpperCamelCase__					,				UpperCamelCase__   :Optional[Any]												= self.text_encoder(
												    input_ids=UpperCamelCase_    ,   attention_mask=UpperCamelCase_							)

												# duplicate unconditional embeddings for each generation per prompt, using mps friendly method

												UpperCamelCase__   :Optional[int]												= negative_prompt_embeds.shape[1]
												UpperCamelCase__   :List[Any]												= negative_prompt_embeds.repeat(1    ,   UpperCamelCase_							)
												UpperCamelCase__   :Any												= negative_prompt_embeds.view(batch_size * num_images_per_prompt    ,   UpperCamelCase_							)

												UpperCamelCase__   :List[str]												= uncond_text_encoder_hidden_states.shape[1]
												UpperCamelCase__   :Tuple												= uncond_text_encoder_hidden_states.repeat(1    ,   UpperCamelCase_    ,   1							)
												UpperCamelCase__   :List[str]												= uncond_text_encoder_hidden_states.view(
												    batch_size * num_images_per_prompt    ,   UpperCamelCase_    ,   -1							)
												UpperCamelCase__   :List[Any]												= uncond_text_mask.repeat_interleave(UpperCamelCase_    ,   dim=0							)

												# done duplicates

												# For classifier free guidance, we need to do two forward passes.
												# Here we concatenate the unconditional and text embeddings into a single batch
												# to avoid doing two forward passes
												UpperCamelCase__   :Tuple												= torch.cat([negative_prompt_embeds, prompt_embeds]							)
												UpperCamelCase__   :Optional[int]												= torch.cat([uncond_text_encoder_hidden_states, text_encoder_hidden_states]							)

												UpperCamelCase__   :int												= torch.cat([uncond_text_mask, text_mask]							)

							return prompt_embeds, text_encoder_hidden_states, text_mask




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_=0							):

							'''simple docstring'''


							if is_accelerate_available():
												from accelerate import cpu_offload
							else:
												raise ImportError('''Please install accelerate via `pip install accelerate`'''							)

							UpperCamelCase__   :int												= torch.device(F'''cuda:{gpu_id}'''							)

							UpperCamelCase__   :Tuple												= [
							    self.unet,
							    self.text_encoder,
							    self.movq,
							]
							for cpu_offloaded_model in models:
												if cpu_offloaded_model is not None:
																	cpu_offload(UpperCamelCase_    ,   UpperCamelCase_							)




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_=0							):

							'''simple docstring'''


							if is_accelerate_available() and is_accelerate_version('''>='''    ,   '''0.17.0.dev0'''							):
												from accelerate import cpu_offload_with_hook
							else:
												raise ImportError('''`enable_model_cpu_offload` requires `accelerate v0.17.0` or higher.'''							)

							UpperCamelCase__   :Tuple												= torch.device(F'''cuda:{gpu_id}'''							)

							if self.device.type != "cpu":
												self.to('''cpu'''    ,   silence_dtype_warnings=UpperCamelCase_							)
												torch.cuda.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)

							UpperCamelCase__   :str												= None
							for cpu_offloaded_model in [self.text_encoder, self.unet, self.movq]:
												UpperCamelCase__					,				UpperCamelCase__   :Union[str, Any]												= cpu_offload_with_hook(UpperCamelCase_    ,   UpperCamelCase_    ,   prev_module_hook=UpperCamelCase_							)

							if self.safety_checker is not None:
												UpperCamelCase__					,				UpperCamelCase__   :Optional[Any]												= cpu_offload_with_hook(self.safety_checker    ,   UpperCamelCase_    ,   prev_module_hook=UpperCamelCase_							)

							# We'll offload the last model manually.
							UpperCamelCase__   :Optional[Any]												= hook




		@property
		# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._execution_device
		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							if not hasattr(self.unet    ,   '''_hf_hook'''							):
												return self.device
							for module in self.unet.modules():
												if (
												    hasattr(UpperCamelCase_    ,   '''_hf_hook'''							)
												    and hasattr(module._hf_hook    ,   '''execution_device'''							)
												    and module._hf_hook.execution_device is not None
												):
																	return torch.device(module._hf_hook.execution_device							)
							return self.device




		@torch.no_grad()
		@replace_example_docstring(UpperCamelCase_							)
		def __call__(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_ = None    ,   UpperCamelCase_ = 512    ,   UpperCamelCase_ = 512    ,   UpperCamelCase_ = 100    ,   UpperCamelCase_ = 4.0    ,   UpperCamelCase_ = 1    ,   UpperCamelCase_ = None    ,   UpperCamelCase_ = None    ,   UpperCamelCase_ = "pil"    ,   UpperCamelCase_ = True    ,   ):

							'''simple docstring'''


							if isinstance(UpperCamelCase_    ,   UpperCamelCase_							):
												UpperCamelCase__   :int												= 1
							elif isinstance(UpperCamelCase_    ,   UpperCamelCase_							):
												UpperCamelCase__   :Optional[int]												= len(UpperCamelCase_							)
							else:
												raise ValueError(F'''`prompt` has to be of type `str` or `list` but is {type(UpperCamelCase_							)}'''							)

							UpperCamelCase__   :Optional[int]												= self._execution_device

							UpperCamelCase__   :Any												= batch_size * num_images_per_prompt
							UpperCamelCase__   :List[Any]												= guidance_scale > 1.0

							UpperCamelCase__					,				UpperCamelCase__					,				UpperCamelCase__   :Optional[Any]												= self._encode_prompt(
							    UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_							)

							if isinstance(UpperCamelCase_    ,   UpperCamelCase_							):
												UpperCamelCase__   :Optional[Any]												= torch.cat(UpperCamelCase_    ,   dim=0							)
							if isinstance(UpperCamelCase_    ,   UpperCamelCase_							):
												UpperCamelCase__   :Optional[int]												= torch.cat(UpperCamelCase_    ,   dim=0							)

							if do_classifier_free_guidance:
												UpperCamelCase__   :int												= image_embeds.repeat_interleave(UpperCamelCase_    ,   dim=0							)
												UpperCamelCase__   :List[Any]												= negative_image_embeds.repeat_interleave(UpperCamelCase_    ,   dim=0							)

							UpperCamelCase__   :List[str]												= torch.cat([negative_image_embeds, image_embeds]    ,   dim=0							).to(
							    dtype=prompt_embeds.dtype    ,   device=UpperCamelCase_							)

							self.scheduler.set_timesteps(UpperCamelCase_    ,   device=UpperCamelCase_							)
							UpperCamelCase__   :int												= self.scheduler.timesteps

							UpperCamelCase__   :List[Any]												= self.unet.config.in_channels

							UpperCamelCase__					,				UpperCamelCase__   :Dict												= get_new_h_w(UpperCamelCase_    ,   UpperCamelCase_    ,   self.movq_scale_factor							)

							# create initial latent
							UpperCamelCase__   :List[Any]												= self.prepare_latents(
							    (batch_size, num_channels_latents, height, width)    ,   text_encoder_hidden_states.dtype    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   self.scheduler    ,   )

							for i, t in enumerate(self.progress_bar(UpperCamelCase_							)							):
												# expand the latents if we are doing classifier free guidance
												UpperCamelCase__   :List[Any]												= torch.cat([latents] * 2							) if do_classifier_free_guidance else latents

												UpperCamelCase__   :Any												= {'''text_embeds''': prompt_embeds, '''image_embeds''': image_embeds}
												UpperCamelCase__   :Optional[int]												= self.unet(
												    sample=UpperCamelCase_    ,   timestep=UpperCamelCase_    ,   encoder_hidden_states=UpperCamelCase_    ,   added_cond_kwargs=UpperCamelCase_    ,   return_dict=UpperCamelCase_    ,   )[0]

												if do_classifier_free_guidance:
																	UpperCamelCase__					,				UpperCamelCase__   :Optional[Any]												= noise_pred.split(latents.shape[1]    ,   dim=1							)
																	UpperCamelCase__					,				UpperCamelCase__   :List[Any]												= noise_pred.chunk(2							)
																	UpperCamelCase__					,				UpperCamelCase__   :Optional[int]												= variance_pred.chunk(2							)
																	UpperCamelCase__   :int												= noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
																	UpperCamelCase__   :Union[str, Any]												= torch.cat([noise_pred, variance_pred_text]    ,   dim=1							)

												if not (
												    hasattr(self.scheduler.config    ,   '''variance_type'''							)
												    and self.scheduler.config.variance_type in ["learned", "learned_range"]
												):
																	UpperCamelCase__					,				UpperCamelCase__   :int												= noise_pred.split(latents.shape[1]    ,   dim=1							)

												# compute the previous noisy sample x_t -> x_t-1
												UpperCamelCase__   :Optional[int]												= self.scheduler.step(
												    UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   generator=UpperCamelCase_    ,   ).prev_sample
							# post-processing
							UpperCamelCase__   :List[Any]												= self.movq.decode(UpperCamelCase_    ,   force_not_quantize=UpperCamelCase_							)['''sample''']

							if output_type not in ["pt", "np", "pil"]:
												raise ValueError(F'''Only the output types `pt`, `pil` and `np` are supported not output_type={output_type}'''							)

							if output_type in ["np", "pil"]:
												UpperCamelCase__   :List[str]												= image * 0.5 + 0.5
												UpperCamelCase__   :Optional[Any]												= image.clamp(0    ,   1							)
												UpperCamelCase__   :List[str]												= image.cpu().permute(0    ,   2    ,   3    ,   1							).float().numpy()

							if output_type == "pil":
												UpperCamelCase__   :str												= self.numpy_to_pil(UpperCamelCase_							)

							if not return_dict:
												return (image,)

							return ImagePipelineOutput(images=UpperCamelCase_							)



'''simple docstring'''
import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def 			a      (			__a ,					__a   )  ->	Optional[int]:



					'''simple docstring'''






					assert isinstance(__a ,					__a   )
					assert dataset.num_rows == 4
					assert dataset.num_columns == 3
					assert dataset.column_names == ["col_1", "col_2", "col_3"]
					for feature, expected_dtype in expected_features.items():
										assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('''keep_in_memory''' ,					[False, True]   )
def 			a      (			__a ,					__a ,					__a   )  ->	Optional[Any]:



					'''simple docstring'''






					UpperCamelCase__   :Union[str, Any]												= tmp_path / '''cache'''
					UpperCamelCase__   :Dict												= {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
					with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
										UpperCamelCase__   :Tuple												= JsonDatasetReader(__a ,					cache_dir=__a ,					keep_in_memory=__a   ).read()
					_check_json_dataset(__a ,					__a   )


@pytest.mark.parametrize(
    '''features''' ,					[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ] ,					)
def 			a      (			__a ,					__a ,					__a   )  ->	Any:



					'''simple docstring'''






					UpperCamelCase__   :Union[str, Any]												= tmp_path / '''cache'''
					UpperCamelCase__   :Optional[Any]												= {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
					UpperCamelCase__   :Optional[Any]												= features.copy() if features else default_expected_features
					UpperCamelCase__   :Tuple												= (
					    Features({feature: Value(__a   ) for feature, dtype in features.items()}   ) if features is not None else None
					)
					UpperCamelCase__   :int												= JsonDatasetReader(__a ,					features=__a ,					cache_dir=__a   ).read()
					_check_json_dataset(__a ,					__a   )


@pytest.mark.parametrize(
    '''features''' ,					[
        None,
        {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''},
    ] ,					)
def 			a      (			__a ,					__a ,					__a   )  ->	Tuple:



					'''simple docstring'''






					UpperCamelCase__   :int												= tmp_path / '''cache'''
					UpperCamelCase__   :str												= {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''}
					UpperCamelCase__   :Any												= features.copy() if features else default_expected_features
					UpperCamelCase__   :Union[str, Any]												= (
					    Features({feature: Value(__a   ) for feature, dtype in features.items()}   ) if features is not None else None
					)
					UpperCamelCase__   :Any												= JsonDatasetReader(__a ,					features=__a ,					cache_dir=__a   ).read()
					assert isinstance(__a ,					__a   )
					assert dataset.num_rows == 2
					assert dataset.num_columns == 3
					assert dataset.column_names == ["col_3", "col_1", "col_2"]
					for feature, expected_dtype in expected_features.items():
										assert dataset.features[feature].dtype == expected_dtype


def 			a      (			__a ,					__a   )  ->	List[Any]:



					'''simple docstring'''






					UpperCamelCase__   :Any												= {'''col_2''': '''int64''', '''col_3''': '''float64''', '''col_1''': '''string'''}
					UpperCamelCase__   :int												= features.copy()
					UpperCamelCase__   :List[Any]												= (
					    Features({feature: Value(__a   ) for feature, dtype in features.items()}   ) if features is not None else None
					)
					UpperCamelCase__   :Optional[int]												= tmp_path / '''cache'''
					UpperCamelCase__   :Dict												= JsonDatasetReader(__a ,					features=__a ,					cache_dir=__a   ).read()
					assert isinstance(__a ,					__a   )
					assert dataset.num_rows == 2
					assert dataset.num_columns == 3
					assert dataset.column_names == ["col_2", "col_3", "col_1"]
					for feature, expected_dtype in expected_features.items():
										assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('''split''' ,					[None, NamedSplit('''train'''   ), '''train''', '''test''']   )
def 			a      (			__a ,					__a ,					__a   )  ->	List[Any]:



					'''simple docstring'''






					UpperCamelCase__   :Union[str, Any]												= tmp_path / '''cache'''
					UpperCamelCase__   :Optional[Any]												= {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
					UpperCamelCase__   :List[Any]												= JsonDatasetReader(__a ,					cache_dir=__a ,					split=__a   ).read()
					_check_json_dataset(__a ,					__a   )
					assert dataset.split == split if split else "train"


@pytest.mark.parametrize('''path_type''' ,					[str, list]   )
def 			a      (			__a ,					__a ,					__a   )  ->	Any:



					'''simple docstring'''






					if issubclass(__a ,					__a   ):
										UpperCamelCase__   :Union[str, Any]												= jsonl_path
					elif issubclass(__a ,					__a   ):
										UpperCamelCase__   :int												= [jsonl_path]
					UpperCamelCase__   :Dict												= tmp_path / '''cache'''
					UpperCamelCase__   :Any												= {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
					UpperCamelCase__   :List[str]												= JsonDatasetReader(__a ,					cache_dir=__a   ).read()
					_check_json_dataset(__a ,					__a   )


def 			a      (			__a ,					__a ,					__a=("train",)   )  ->	Optional[Any]:



					'''simple docstring'''






					assert isinstance(__a ,					__a   )
					for split in splits:
										UpperCamelCase__   :Optional[int]												= dataset_dict[split]
										assert dataset.num_rows == 4
										assert dataset.num_columns == 3
										assert dataset.column_names == ["col_1", "col_2", "col_3"]
										for feature, expected_dtype in expected_features.items():
															assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('''keep_in_memory''' ,					[False, True]   )
def 			a      (			__a ,					__a ,					__a   )  ->	List[str]:



					'''simple docstring'''






					UpperCamelCase__   :List[str]												= tmp_path / '''cache'''
					UpperCamelCase__   :Dict												= {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
					with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
										UpperCamelCase__   :str												= JsonDatasetReader({'''train''': jsonl_path} ,					cache_dir=__a ,					keep_in_memory=__a   ).read()
					_check_json_datasetdict(__a ,					__a   )


@pytest.mark.parametrize(
    '''features''' ,					[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ] ,					)
def 			a      (			__a ,					__a ,					__a   )  ->	int:



					'''simple docstring'''






					UpperCamelCase__   :Tuple												= tmp_path / '''cache'''
					UpperCamelCase__   :Any												= {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
					UpperCamelCase__   :Optional[int]												= features.copy() if features else default_expected_features
					UpperCamelCase__   :str												= (
					    Features({feature: Value(__a   ) for feature, dtype in features.items()}   ) if features is not None else None
					)
					UpperCamelCase__   :Dict												= JsonDatasetReader({'''train''': jsonl_path} ,					features=__a ,					cache_dir=__a   ).read()
					_check_json_datasetdict(__a ,					__a   )


@pytest.mark.parametrize('''split''' ,					[None, NamedSplit('''train'''   ), '''train''', '''test''']   )
def 			a      (			__a ,					__a ,					__a   )  ->	str:



					'''simple docstring'''






					if split:
										UpperCamelCase__   :List[str]												= {split: jsonl_path}
					else:
										UpperCamelCase__   :int												= '''train'''
										UpperCamelCase__   :int												= {'''train''': jsonl_path, '''test''': jsonl_path}
					UpperCamelCase__   :Any												= tmp_path / '''cache'''
					UpperCamelCase__   :Union[str, Any]												= {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
					UpperCamelCase__   :Any												= JsonDatasetReader(__a ,					cache_dir=__a   ).read()
					_check_json_datasetdict(__a ,					__a ,					splits=list(path.keys()   )   )
					assert all(dataset[split].split == split for split in path.keys()   )


def 			a      (			__a   )  ->	Union[str, Any]:



					'''simple docstring'''






					return json.load(__a   )


def 			a      (			__a   )  ->	int:



					'''simple docstring'''






					return [json.loads(__a   ) for line in buffer]







class    lowercase	:

		"""simple docstring"""

		@pytest.mark.parametrize('''lines, load_json_function'''    ,   [(True, load_json_lines), (False, load_json)]							)
		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_							):

							'''simple docstring'''


							with io.BytesIO() as buffer:
												JsonDatasetWriter(UpperCamelCase_    ,   UpperCamelCase_    ,   lines=UpperCamelCase_							).write()
												buffer.seek(0							)
												UpperCamelCase__   :List[Any]												= load_json_function(UpperCamelCase_							)
							assert isinstance(UpperCamelCase_    ,   UpperCamelCase_							)
							assert isinstance(exported_content[0]    ,   UpperCamelCase_							)
							assert len(UpperCamelCase_							) == 10




		@pytest.mark.parametrize(
		    '''orient, container, keys, len_at'''    ,   [
		        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
		        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
		        ('''index''', dict, set('''0123456789'''							), None),
		        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
		        ('''values''', list, None, None),
		        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
		    ]    ,   )
		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_							):

							'''simple docstring'''


							with io.BytesIO() as buffer:
												JsonDatasetWriter(UpperCamelCase_    ,   UpperCamelCase_    ,   lines=UpperCamelCase_    ,   orient=UpperCamelCase_							).write()
												buffer.seek(0							)
												UpperCamelCase__   :Optional[int]												= load_json(UpperCamelCase_							)
							assert isinstance(UpperCamelCase_    ,   UpperCamelCase_							)
							if keys:
												if container is dict:
																	assert exported_content.keys() == keys
												else:
																	assert exported_content[0].keys() == keys
							else:
												assert not hasattr(UpperCamelCase_    ,   '''keys'''							) and not hasattr(exported_content[0]    ,   '''keys'''							)
							if len_at:
												assert len(exported_content[len_at]							) == 10
							else:
												assert len(UpperCamelCase_							) == 10




		@pytest.mark.parametrize('''lines, load_json_function'''    ,   [(True, load_json_lines), (False, load_json)]							)
		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_							):

							'''simple docstring'''


							with io.BytesIO() as buffer:
												JsonDatasetWriter(UpperCamelCase_    ,   UpperCamelCase_    ,   lines=UpperCamelCase_    ,   num_proc=2							).write()
												buffer.seek(0							)
												UpperCamelCase__   :Union[str, Any]												= load_json_function(UpperCamelCase_							)
							assert isinstance(UpperCamelCase_    ,   UpperCamelCase_							)
							assert isinstance(exported_content[0]    ,   UpperCamelCase_							)
							assert len(UpperCamelCase_							) == 10




		@pytest.mark.parametrize(
		    '''orient, container, keys, len_at'''    ,   [
		        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
		        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
		        ('''index''', dict, set('''0123456789'''							), None),
		        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
		        ('''values''', list, None, None),
		        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
		    ]    ,   )
		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_							):

							'''simple docstring'''


							with io.BytesIO() as buffer:
												JsonDatasetWriter(UpperCamelCase_    ,   UpperCamelCase_    ,   lines=UpperCamelCase_    ,   orient=UpperCamelCase_    ,   num_proc=2							).write()
												buffer.seek(0							)
												UpperCamelCase__   :int												= load_json(UpperCamelCase_							)
							assert isinstance(UpperCamelCase_    ,   UpperCamelCase_							)
							if keys:
												if container is dict:
																	assert exported_content.keys() == keys
												else:
																	assert exported_content[0].keys() == keys
							else:
												assert not hasattr(UpperCamelCase_    ,   '''keys'''							) and not hasattr(exported_content[0]    ,   '''keys'''							)
							if len_at:
												assert len(exported_content[len_at]							) == 10
							else:
												assert len(UpperCamelCase_							) == 10




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_							):

							'''simple docstring'''


							with pytest.raises(UpperCamelCase_							):
												with io.BytesIO() as buffer:
																	JsonDatasetWriter(UpperCamelCase_    ,   UpperCamelCase_    ,   num_proc=0							)




		@pytest.mark.parametrize('''compression, extension'''    ,   [('''gzip''', '''gz'''), ('''bz2''', '''bz2'''), ('''xz''', '''xz''')]							)
		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__   :Tuple												= tmp_path_factory.mktemp('''data'''							) / F'''test.json.{extension}'''
							UpperCamelCase__   :Union[str, Any]												= str(shared_datadir / F'''test_file.json.{extension}'''							)
							JsonDatasetWriter(UpperCamelCase_    ,   UpperCamelCase_    ,   compression=UpperCamelCase_							).write()

							with fsspec.open(UpperCamelCase_    ,   '''rb'''    ,   compression='''infer'''							) as f:
												UpperCamelCase__   :Dict												= f.read()
							with fsspec.open(UpperCamelCase_    ,   '''rb'''    ,   compression='''infer'''							) as f:
												UpperCamelCase__   :int												= f.read()
							assert exported_content == original_content