





import unittest

import numpy as np

from transformers.testing_utils import require_pytesseract, require_torch
from transformers.utils import is_pytesseract_available, is_torch_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
     import torch

if is_pytesseract_available():
     from PIL import Image

     from transformers import LayoutLMvaImageProcessor



class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  def __init__(				self :		Any					,  lowerCAmelCase_ :		Optional[Any]					,  lowerCAmelCase_ :		Tuple=7					,  lowerCAmelCase_ :		Union[str, Any]=3					,  lowerCAmelCase_ :		List[Any]=1_8					,  lowerCAmelCase_ :		Dict=3_0					,  lowerCAmelCase_ :		Tuple=4_0_0					,  lowerCAmelCase_ :		Optional[int]=True					,  lowerCAmelCase_ :		Dict=None					,  lowerCAmelCase_ :		Any=True					,  )							->						Tuple:
     __lowerCAmelCase								=	size if size is not None else {'height': 1_8, 'width': 1_8}
     __lowerCAmelCase								=	parent
     __lowerCAmelCase								=	batch_size
     __lowerCAmelCase								=	num_channels
     __lowerCAmelCase								=	image_size
     __lowerCAmelCase								=	min_resolution
     __lowerCAmelCase								=	max_resolution
     __lowerCAmelCase								=	do_resize
     __lowerCAmelCase								=	size
     __lowerCAmelCase								=	apply_ocr






  def 				lowercase		(				self :		Any     )							->						str:
     return {"do_resize": self.do_resize, "size": self.size, "apply_ocr": self.apply_ocr}





@require_torch
@require_pytesseract
class 				_UpperCAmelCase    (    _UpperCamelCase		,    unittest.TestCase    ):

  """simple docstring"""


  a_        = LayoutLMvaImageProcessor if is_pytesseract_available() else None






  def 				lowercase		(				self :		Dict     )							->						Tuple:
     __lowerCAmelCase								=	LayoutLMvaImageProcessingTester(self     )






  @property
  def 				lowercase		(				self :		Optional[int]     )							->						Optional[Any]:
     return self.image_processor_tester.prepare_image_processor_dict()






  def 				lowercase		(				self :		Optional[int]     )							->						str:
     __lowerCAmelCase								=	self.image_processing_class(**self.image_processor_dict     )
     self.assertTrue(hasattr(lowerCAmelCase_					,  'do_resize'     )     )
     self.assertTrue(hasattr(lowerCAmelCase_					,  'size'     )     )
     self.assertTrue(hasattr(lowerCAmelCase_					,  'apply_ocr'     )     )






  def 				lowercase		(				self :		List[Any]     )							->						Union[str, Any]:
     __lowerCAmelCase								=	self.image_processing_class.from_dict(self.image_processor_dict     )
     self.assertEqual(image_processor.size					,  {'height': 1_8, 'width': 1_8}     )

     __lowerCAmelCase								=	self.image_processing_class.from_dict(self.image_processor_dict					,  size=4_2     )
     self.assertEqual(image_processor.size					,  {'height': 4_2, 'width': 4_2}     )






  def 				lowercase		(				self :		Optional[int]     )							->						str:
     pass






  def 				lowercase		(				self :		List[Any]     )							->						str:
     # Initialize image_processing
     __lowerCAmelCase								=	self.image_processing_class(**self.image_processor_dict     )
     # create random PIL images
     __lowerCAmelCase								=	prepare_image_inputs(self.image_processor_tester					,  equal_resolution=lowerCAmelCase_     )
     for image in image_inputs:
        self.assertIsInstance(lowerCAmelCase_					,  Image.Image     )

     # Test not batched input
     __lowerCAmelCase								=	image_processing(image_inputs[0]					,  return_tensors='pt'     )
     self.assertEqual(
         encoding.pixel_values.shape					,  (
             1,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.size['height'],
             self.image_processor_tester.size['width'],
         )					,  )

     self.assertIsInstance(encoding.words					,  lowerCAmelCase_     )
     self.assertIsInstance(encoding.boxes					,  lowerCAmelCase_     )

     # Test batched
     __lowerCAmelCase								=	image_processing(lowerCAmelCase_					,  return_tensors='pt'     ).pixel_values
     self.assertEqual(
         encoded_images.shape					,  (
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.size['height'],
             self.image_processor_tester.size['width'],
         )					,  )






  def 				lowercase		(				self :		str     )							->						Optional[int]:
     # Initialize image_processing
     __lowerCAmelCase								=	self.image_processing_class(**self.image_processor_dict     )
     # create random numpy tensors
     __lowerCAmelCase								=	prepare_image_inputs(self.image_processor_tester					,  equal_resolution=lowerCAmelCase_					,  numpify=lowerCAmelCase_     )
     for image in image_inputs:
        self.assertIsInstance(lowerCAmelCase_					,  np.ndarray     )

     # Test not batched input
     __lowerCAmelCase								=	image_processing(image_inputs[0]					,  return_tensors='pt'     ).pixel_values
     self.assertEqual(
         encoded_images.shape					,  (
             1,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.size['height'],
             self.image_processor_tester.size['width'],
         )					,  )

     # Test batched
     __lowerCAmelCase								=	image_processing(lowerCAmelCase_					,  return_tensors='pt'     ).pixel_values
     self.assertEqual(
         encoded_images.shape					,  (
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.size['height'],
             self.image_processor_tester.size['width'],
         )					,  )






  def 				lowercase		(				self :		Any     )							->						Tuple:
     # Initialize image_processing
     __lowerCAmelCase								=	self.image_processing_class(**self.image_processor_dict     )
     # create random PyTorch tensors
     __lowerCAmelCase								=	prepare_image_inputs(self.image_processor_tester					,  equal_resolution=lowerCAmelCase_					,  torchify=lowerCAmelCase_     )
     for image in image_inputs:
        self.assertIsInstance(lowerCAmelCase_					,  torch.Tensor     )

     # Test not batched input
     __lowerCAmelCase								=	image_processing(image_inputs[0]					,  return_tensors='pt'     ).pixel_values
     self.assertEqual(
         encoded_images.shape					,  (
             1,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.size['height'],
             self.image_processor_tester.size['width'],
         )					,  )

     # Test batched
     __lowerCAmelCase								=	image_processing(lowerCAmelCase_					,  return_tensors='pt'     ).pixel_values
     self.assertEqual(
         encoded_images.shape					,  (
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.size['height'],
             self.image_processor_tester.size['width'],
         )					,  )






  def 				lowercase		(				self :		List[str]     )							->						int:
     # with apply_OCR = True
     __lowerCAmelCase								=	LayoutLMvaImageProcessor()

     from datasets import load_dataset

     __lowerCAmelCase								=	load_dataset('hf-internal-testing/fixtures_docvqa'					,  split='test'     )

     __lowerCAmelCase								=	Image.open(ds[0]['file']     ).convert('RGB'     )

     __lowerCAmelCase								=	image_processing(lowerCAmelCase_					,  return_tensors='pt'     )

     self.assertEqual(encoding.pixel_values.shape					,  (1, 3, 2_2_4, 2_2_4)     )
     self.assertEqual(len(encoding.words     )					,  len(encoding.boxes     )     )

     # fmt: off
     # the words and boxes were obtained with Tesseract 4.1.1
     __lowerCAmelCase								=	[['11:14', 'to', '11:39', 'a.m', '11:39', 'to', '11:44', 'a.m.', '11:44', 'a.m.', 'to', '12:25', 'p.m.', '12:25', 'to', '12:58', 'p.m.', '12:58', 'to', '4:00', 'p.m.', '2:00', 'to', '5:00', 'p.m.', 'Coffee', 'Break', 'Coffee', 'will', 'be', 'served', 'for', 'men', 'and', 'women', 'in', 'the', 'lobby', 'adjacent', 'to', 'exhibit', 'area.', 'Please', 'move', 'into', 'exhibit', 'area.', '(Exhibits', 'Open)', 'TRRF', 'GENERAL', 'SESSION', '(PART', '|)', 'Presiding:', 'Lee', 'A.', 'Waller', 'TRRF', 'Vice', 'President', '“Introductory', 'Remarks”', 'Lee', 'A.', 'Waller,', 'TRRF', 'Vice', 'Presi-', 'dent', 'Individual', 'Interviews', 'with', 'TRRF', 'Public', 'Board', 'Members', 'and', 'Sci-', 'entific', 'Advisory', 'Council', 'Mem-', 'bers', 'Conducted', 'by', 'TRRF', 'Treasurer', 'Philip', 'G.', 'Kuehn', 'to', 'get', 'answers', 'which', 'the', 'public', 'refrigerated', 'warehousing', 'industry', 'is', 'looking', 'for.', 'Plus', 'questions', 'from', 'the', 'floor.', 'Dr.', 'Emil', 'M.', 'Mrak,', 'University', 'of', 'Cal-', 'ifornia,', 'Chairman,', 'TRRF', 'Board;', 'Sam', 'R.', 'Cecil,', 'University', 'of', 'Georgia', 'College', 'of', 'Agriculture;', 'Dr.', 'Stanley', 'Charm,', 'Tufts', 'University', 'School', 'of', 'Medicine;', 'Dr.', 'Robert', 'H.', 'Cotton,', 'ITT', 'Continental', 'Baking', 'Company;', 'Dr.', 'Owen', 'Fennema,', 'University', 'of', 'Wis-', 'consin;', 'Dr.', 'Robert', 'E.', 'Hardenburg,', 'USDA.', 'Questions', 'and', 'Answers', 'Exhibits', 'Open', 'Capt.', 'Jack', 'Stoney', 'Room', 'TRRF', 'Scientific', 'Advisory', 'Council', 'Meeting', 'Ballroom', 'Foyer']]  # noqa: E231
     __lowerCAmelCase								=	[[[1_4_1, 5_7, 2_1_4, 6_9], [2_2_8, 5_8, 2_5_2, 6_9], [1_4_1, 7_5, 2_1_6, 8_8], [2_3_0, 7_9, 2_8_0, 8_8], [1_4_2, 2_6_0, 2_1_8, 2_7_3], [2_3_0, 2_6_1, 2_5_5, 2_7_3], [1_4_3, 2_7_9, 2_1_8, 2_9_0], [2_3_1, 2_8_2, 2_9_0, 2_9_1], [1_4_3, 3_4_2, 2_1_8, 3_5_4], [2_3_1, 3_4_5, 2_8_9, 3_5_5], [2_0_2, 3_6_2, 2_2_7, 3_7_3], [1_4_3, 3_7_9, 2_2_0, 3_9_2], [2_3_1, 3_8_2, 2_9_1, 3_9_4], [1_4_4, 7_1_4, 2_2_0, 7_2_6], [2_3_1, 7_1_5, 2_5_6, 7_2_6], [1_4_4, 7_3_2, 2_2_0, 7_4_5], [2_3_2, 7_3_6, 2_9_1, 7_4_7], [1_4_4, 7_6_9, 2_1_8, 7_8_2], [2_3_1, 7_7_0, 2_5_6, 7_8_2], [1_4_1, 7_8_8, 2_0_2, 8_0_1], [2_1_5, 7_9_1, 2_7_4, 8_0_4], [1_4_3, 8_2_6, 2_0_4, 8_3_8], [2_1_5, 8_2_6, 2_4_0, 8_3_8], [1_4_2, 8_4_4, 2_0_2, 8_5_7], [2_1_5, 8_4_7, 2_7_4, 8_5_9], [3_3_4, 5_7, 4_2_7, 6_9], [4_4_0, 5_7, 5_2_2, 6_9], [3_6_9, 7_5, 4_6_1, 8_8], [4_6_9, 7_5, 5_1_6, 8_8], [5_2_8, 7_6, 5_6_2, 8_8], [5_7_0, 7_6, 6_6_7, 8_8], [6_7_5, 7_5, 7_1_1, 8_7], [7_2_1, 7_9, 7_7_8, 8_8], [7_8_9, 7_5, 8_4_0, 8_8], [3_6_9, 9_7, 4_7_0, 1_0_7], [4_8_4, 9_4, 5_0_7, 1_0_6], [5_1_8, 9_4, 5_6_2, 1_0_7], [5_7_6, 9_4, 6_5_5, 1_1_0], [6_6_8, 9_4, 7_9_2, 1_0_9], [8_0_4, 9_5, 8_2_9, 1_0_7], [3_6_9, 1_1_3, 4_6_5, 1_2_5], [4_7_7, 1_1_6, 5_4_7, 1_2_5], [5_6_2, 1_1_3, 6_5_8, 1_2_5], [6_7_1, 1_1_6, 7_4_8, 1_2_5], [7_6_1, 1_1_3, 8_1_1, 1_2_5], [3_6_9, 1_3_1, 4_6_5, 1_4_3], [4_7_7, 1_3_3, 5_4_8, 1_4_3], [5_6_3, 1_3_0, 6_9_8, 1_4_5], [7_1_0, 1_3_0, 8_0_2, 1_4_6], [3_3_6, 1_7_1, 4_1_2, 1_8_3], [4_2_3, 1_7_1, 5_7_2, 1_8_3], [5_8_2, 1_7_0, 7_1_6, 1_8_4], [7_2_8, 1_7_1, 8_1_7, 1_8_7], [8_2_9, 1_7_1, 8_4_4, 1_8_6], [3_3_8, 1_9_7, 4_8_2, 2_1_2], [5_0_7, 1_9_6, 5_5_7, 2_0_9], [5_6_9, 1_9_6, 5_9_5, 2_0_8], [6_1_0, 1_9_6, 7_0_2, 2_0_9], [5_0_5, 2_1_4, 5_8_3, 2_2_6], [5_9_5, 2_1_4, 6_5_6, 2_2_7], [6_7_0, 2_1_5, 8_0_7, 2_2_7], [3_3_5, 2_5_9, 5_4_3, 2_7_4], [5_5_6, 2_5_9, 7_0_8, 2_7_2], [3_7_2, 2_7_9, 4_2_2, 2_9_1], [4_3_5, 2_7_9, 4_6_0, 2_9_1], [4_7_4, 2_7_9, 5_7_4, 2_9_2], [5_8_7, 2_7_8, 6_6_4, 2_9_1], [6_7_6, 2_7_8, 7_3_8, 2_9_1], [7_5_1, 2_7_9, 8_3_4, 2_9_1], [3_7_2, 2_9_8, 4_3_4, 3_1_0], [3_3_5, 3_4_1, 4_8_3, 3_5_4], [4_9_7, 3_4_1, 6_5_5, 3_5_4], [6_6_7, 3_4_1, 7_2_8, 3_5_4], [7_4_0, 3_4_1, 8_2_5, 3_5_4], [3_3_5, 3_6_0, 4_3_0, 3_7_2], [4_4_2, 3_6_0, 5_3_4, 3_7_2], [5_4_5, 3_5_9, 6_8_7, 3_7_2], [6_9_7, 3_6_0, 7_5_4, 3_7_2], [7_6_5, 3_6_0, 8_2_3, 3_7_3], [3_3_4, 3_7_8, 4_2_8, 3_9_1], [4_4_0, 3_7_8, 5_7_7, 3_9_4], [5_9_0, 3_7_8, 7_0_5, 3_9_1], [7_2_0, 3_7_8, 8_0_1, 3_9_1], [3_3_4, 3_9_7, 4_0_0, 4_0_9], [3_7_0, 4_1_6, 5_2_9, 4_2_9], [5_4_4, 4_1_6, 5_7_6, 4_3_2], [5_8_7, 4_1_6, 6_6_5, 4_2_8], [6_7_7, 4_1_6, 8_1_4, 4_2_9], [3_7_2, 4_3_5, 4_5_2, 4_5_0], [4_6_5, 4_3_4, 4_9_5, 4_4_7], [5_1_1, 4_3_4, 6_0_0, 4_4_7], [6_1_1, 4_3_6, 6_3_7, 4_4_7], [6_4_9, 4_3_6, 6_9_4, 4_5_1], [7_0_5, 4_3_8, 8_2_4, 4_4_7], [3_6_9, 4_5_3, 4_5_2, 4_6_6], [4_6_4, 4_5_4, 5_0_9, 4_6_6], [5_2_2, 4_5_3, 6_1_1, 4_6_9], [6_2_5, 4_5_3, 7_9_2, 4_6_9], [3_7_0, 4_7_2, 5_5_6, 4_8_8], [5_7_0, 4_7_2, 6_8_4, 4_8_7], [6_9_7, 4_7_2, 7_1_8, 4_8_5], [7_3_2, 4_7_2, 8_3_5, 4_8_8], [3_6_9, 4_9_0, 4_1_1, 5_0_3], [4_2_5, 4_9_0, 4_8_4, 5_0_3], [4_9_6, 4_9_0, 6_3_5, 5_0_6], [6_4_5, 4_9_0, 7_0_7, 5_0_3], [7_1_8, 4_9_1, 7_6_1, 5_0_3], [7_7_1, 4_9_0, 8_4_0, 5_0_3], [3_3_6, 5_1_0, 3_7_4, 5_2_1], [3_8_8, 5_1_0, 4_4_7, 5_2_2], [4_6_0, 5_1_0, 4_8_9, 5_2_1], [5_0_3, 5_1_0, 5_8_0, 5_2_2], [5_9_2, 5_0_9, 7_3_6, 5_2_5], [7_4_5, 5_0_9, 7_7_0, 5_2_2], [7_8_1, 5_0_9, 8_4_0, 5_2_2], [3_3_8, 5_2_8, 4_3_4, 5_4_1], [4_4_8, 5_2_8, 5_9_6, 5_4_1], [6_0_9, 5_2_7, 6_8_7, 5_4_0], [7_0_0, 5_2_8, 7_9_2, 5_4_1], [3_3_6, 5_4_6, 3_9_7, 5_5_9], [4_0_7, 5_4_6, 4_3_1, 5_5_9], [4_4_3, 5_4_6, 5_2_5, 5_6_0], [5_3_7, 5_4_6, 6_8_0, 5_6_2], [6_8_8, 5_4_6, 7_1_4, 5_5_9], [7_2_2, 5_4_6, 8_3_7, 5_6_2], [3_3_6, 5_6_5, 4_4_9, 5_8_1], [4_6_1, 5_6_5, 4_8_5, 5_7_7], [4_9_7, 5_6_5, 6_6_5, 5_8_1], [6_8_1, 5_6_5, 7_1_8, 5_7_7], [7_3_2, 5_6_5, 8_3_7, 5_8_0], [3_3_7, 5_8_4, 4_3_8, 5_9_7], [4_5_2, 5_8_3, 5_2_1, 5_9_6], [5_3_5, 5_8_4, 6_7_7, 5_9_9], [6_9_0, 5_8_3, 7_8_7, 5_9_6], [8_0_1, 5_8_3, 8_2_5, 5_9_6], [3_3_8, 6_0_2, 4_7_8, 6_1_5], [4_9_2, 6_0_2, 5_3_0, 6_1_4], [5_4_3, 6_0_2, 6_3_8, 6_1_5], [6_5_0, 6_0_2, 6_7_6, 6_1_4], [6_8_8, 6_0_2, 7_8_8, 6_1_5], [8_0_2, 6_0_2, 8_4_3, 6_1_4], [3_3_7, 6_2_1, 5_0_2, 6_3_3], [5_1_6, 6_2_1, 6_1_5, 6_3_7], [6_2_9, 6_2_1, 7_7_4, 6_3_6], [7_8_9, 6_2_1, 8_2_7, 6_3_3], [3_3_7, 6_3_9, 4_1_8, 6_5_2], [4_3_2, 6_4_0, 5_7_1, 6_5_3], [5_8_7, 6_3_9, 7_3_1, 6_5_5], [7_4_3, 6_3_9, 7_6_9, 6_5_2], [7_8_0, 6_3_9, 8_4_1, 6_5_2], [3_3_8, 6_5_8, 4_4_0, 6_7_3], [4_5_5, 6_5_8, 4_9_1, 6_7_0], [5_0_8, 6_5_8, 6_0_2, 6_7_1], [6_1_6, 6_5_8, 6_3_8, 6_7_0], [6_5_4, 6_5_8, 8_3_5, 6_7_4], [3_3_7, 6_7_7, 4_2_9, 6_8_9], [3_3_7, 7_1_4, 4_8_2, 7_2_6], [4_9_5, 7_1_4, 5_4_8, 7_2_6], [5_6_1, 7_1_4, 6_8_3, 7_2_6], [3_3_8, 7_7_0, 4_6_1, 7_8_2], [4_7_4, 7_6_9, 5_5_4, 7_8_5], [4_8_9, 7_8_8, 5_6_2, 8_0_3], [5_7_6, 7_8_8, 6_4_3, 8_0_1], [6_5_6, 7_8_7, 7_5_1, 8_0_4], [7_6_4, 7_8_8, 8_4_4, 8_0_1], [3_3_4, 8_2_5, 4_2_1, 8_3_8], [4_3_0, 8_2_4, 5_7_4, 8_3_8], [5_8_4, 8_2_4, 7_2_3, 8_4_1], [3_3_5, 8_4_4, 4_5_0, 8_5_7], [4_6_4, 8_4_3, 5_8_3, 8_6_0], [6_2_8, 8_6_2, 7_5_5, 8_7_5], [7_6_9, 8_6_1, 8_4_8, 8_7_8]]]  # noqa: E231
     # fmt: on

     self.assertListEqual(encoding.words					,  lowerCAmelCase_     )
     self.assertListEqual(encoding.boxes					,  lowerCAmelCase_     )

     # with apply_OCR = False
     __lowerCAmelCase								=	LayoutLMvaImageProcessor(apply_ocr=lowerCAmelCase_     )

     __lowerCAmelCase								=	image_processing(lowerCAmelCase_					,  return_tensors='pt'     )

     self.assertEqual(encoding.pixel_values.shape					,  (1, 3, 2_2_4, 2_2_4)     )








import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


_snake_case  :    Dict						    =  pytest.mark.integration



@require_faiss
class 				_UpperCAmelCase    (    _UpperCamelCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		List[Any]     )							->						Optional[Any]:
     __lowerCAmelCase								=	Dataset.from_dict({'filename': ['my_name-train' + '_' + str(lowerCAmelCase_     ) for x in np.arange(3_0     ).tolist()]}     )
     return dset






  def 				lowercase		(				self :		List[str]     )							->						Tuple:
     import faiss

     __lowerCAmelCase								=	self._create_dummy_dataset()
     __lowerCAmelCase								=	dset.map(
         lambda lowerCAmelCase_					,  lowerCAmelCase_     : {"vecs": i * np.ones(5					,  dtype=np.floataa     )}					,  with_indices=lowerCAmelCase_					,  keep_in_memory=lowerCAmelCase_     )
     __lowerCAmelCase								=	dset.add_faiss_index('vecs'					,  batch_size=1_0_0					,  metric_type=faiss.METRIC_INNER_PRODUCT     )
     __lowerCAmelCase	,	__lowerCAmelCase								=	dset.get_nearest_examples('vecs'					,  np.ones(5					,  dtype=np.floataa     )     )
     self.assertEqual(examples['filename'][0]					,  'my_name-train_29'     )
     dset.drop_index('vecs'     )






  def 				lowercase		(				self :		Optional[Any]     )							->						str:
     import faiss

     __lowerCAmelCase								=	self._create_dummy_dataset()
     dset.add_faiss_index_from_external_arrays(
         external_arrays=np.ones((3_0, 5)     ) * np.arange(3_0     ).reshape(-1					,  1     )					,  index_name='vecs'					,  batch_size=1_0_0					,  metric_type=faiss.METRIC_INNER_PRODUCT					,  )
     __lowerCAmelCase	,	__lowerCAmelCase								=	dset.get_nearest_examples('vecs'					,  np.ones(5					,  dtype=np.floataa     )     )
     self.assertEqual(examples['filename'][0]					,  'my_name-train_29'     )






  def 				lowercase		(				self :		int     )							->						Optional[Any]:
     import faiss

     __lowerCAmelCase								=	self._create_dummy_dataset()
     dset.add_faiss_index_from_external_arrays(
         external_arrays=np.ones((3_0, 5)     ) * np.arange(3_0     ).reshape(-1					,  1     )					,  index_name='vecs'					,  metric_type=faiss.METRIC_INNER_PRODUCT					,  )

     # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
     # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
     # see https://bugs.python.org/issue14243 and
     # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
     with tempfile.NamedTemporaryFile(delete=lowerCAmelCase_     ) as tmp_file:
        dset.save_faiss_index('vecs'					,  tmp_file.name     )
        dset.load_faiss_index('vecs2'					,  tmp_file.name     )
     os.unlink(tmp_file.name     )

     __lowerCAmelCase	,	__lowerCAmelCase								=	dset.get_nearest_examples('vecs2'					,  np.ones(5					,  dtype=np.floataa     )     )
     self.assertEqual(examples['filename'][0]					,  'my_name-train_29'     )






  def 				lowercase		(				self :		Union[str, Any]     )							->						List[Any]:
     __lowerCAmelCase								=	self._create_dummy_dataset()
     dset.add_faiss_index_from_external_arrays(
         external_arrays=np.ones((3_0, 5)     ) * np.arange(3_0     ).reshape(-1					,  1     )					,  index_name='vecs'     )
     dset.drop_index('vecs'     )
     self.assertRaises(lowerCAmelCase_					,  partial(dset.get_nearest_examples					,  'vecs2'					,  np.ones(5					,  dtype=np.floataa     )     )     )






  def 				lowercase		(				self :		Union[str, Any]     )							->						Tuple:
     from elasticsearch import Elasticsearch

     __lowerCAmelCase								=	self._create_dummy_dataset()
     with patch('elasticsearch.Elasticsearch.search'     ) as mocked_search, patch(
         'elasticsearch.client.IndicesClient.create'     ) as mocked_index_create, patch('elasticsearch.helpers.streaming_bulk'     ) as mocked_bulk:
        __lowerCAmelCase								=	{'acknowledged': True}
        mocked_bulk.return_value([(True, None)] * 3_0     )
        __lowerCAmelCase								=	{'hits': {'hits': [{'_score': 1, '_id': 2_9}]}}
        __lowerCAmelCase								=	Elasticsearch()

        dset.add_elasticsearch_index('filename'					,  es_client=lowerCAmelCase_     )
        __lowerCAmelCase	,	__lowerCAmelCase								=	dset.get_nearest_examples('filename'					,  'my_name-train_29'     )
        self.assertEqual(examples['filename'][0]					,  'my_name-train_29'     )





@require_faiss
class 				_UpperCAmelCase    (    _UpperCamelCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		str     )							->						int:
     import faiss

     __lowerCAmelCase								=	FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT     )

     # add vectors
     index.add_vectors(np.eye(5					,  dtype=np.floataa     )     )
     self.assertIsNotNone(index.faiss_index     )
     self.assertEqual(index.faiss_index.ntotal					,  5     )
     index.add_vectors(np.zeros((5, 5)					,  dtype=np.floataa     )     )
     self.assertEqual(index.faiss_index.ntotal					,  1_0     )

     # single query
     __lowerCAmelCase								=	np.zeros(5					,  dtype=np.floataa     )
     __lowerCAmelCase								=	1
     __lowerCAmelCase	,	__lowerCAmelCase								=	index.search(lowerCAmelCase_     )
     self.assertRaises(lowerCAmelCase_					,  index.search					,  query.reshape(-1					,  1     )     )
     self.assertGreater(scores[0]					,  0     )
     self.assertEqual(indices[0]					,  1     )

     # batched queries
     __lowerCAmelCase								=	np.eye(5					,  dtype=np.floataa     )[::-1]
     __lowerCAmelCase	,	__lowerCAmelCase								=	index.search_batch(lowerCAmelCase_     )
     self.assertRaises(lowerCAmelCase_					,  index.search_batch					,  queries[0]     )
     __lowerCAmelCase								=	[scores[0] for scores in total_scores]
     __lowerCAmelCase								=	[indices[0] for indices in total_indices]
     self.assertGreater(np.min(lowerCAmelCase_     )					,  0     )
     self.assertListEqual([4, 3, 2, 1, 0]					,  lowerCAmelCase_     )






  def 				lowercase		(				self :		List[Any]     )							->						List[str]:
     import faiss

     __lowerCAmelCase								=	FaissIndex(string_factory='Flat'     )
     index.add_vectors(np.eye(5					,  dtype=np.floataa     )     )
     self.assertIsInstance(index.faiss_index					,  faiss.IndexFlat     )
     __lowerCAmelCase								=	FaissIndex(string_factory='LSH'     )
     index.add_vectors(np.eye(5					,  dtype=np.floataa     )     )
     self.assertIsInstance(index.faiss_index					,  faiss.IndexLSH     )
     with self.assertRaises(lowerCAmelCase_     ):
        __lowerCAmelCase								=	FaissIndex(string_factory='Flat'					,  custom_index=faiss.IndexFlat(5     )     )






  def 				lowercase		(				self :		Union[str, Any]     )							->						Dict:
     import faiss

     __lowerCAmelCase								=	faiss.IndexFlat(5     )
     __lowerCAmelCase								=	FaissIndex(custom_index=lowerCAmelCase_     )
     index.add_vectors(np.eye(5					,  dtype=np.floataa     )     )
     self.assertIsInstance(index.faiss_index					,  faiss.IndexFlat     )






  def 				lowercase		(				self :		str     )							->						Any:
     import faiss

     __lowerCAmelCase								=	FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT     )
     index.add_vectors(np.eye(5					,  dtype=np.floataa     )     )

     # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
     # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
     # see https://bugs.python.org/issue14243 and
     # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
     with tempfile.NamedTemporaryFile(delete=lowerCAmelCase_     ) as tmp_file:
        index.save(tmp_file.name     )
        __lowerCAmelCase								=	FaissIndex.load(tmp_file.name     )
     os.unlink(tmp_file.name     )

     __lowerCAmelCase								=	np.zeros(5					,  dtype=np.floataa     )
     __lowerCAmelCase								=	1
     __lowerCAmelCase	,	__lowerCAmelCase								=	index.search(lowerCAmelCase_     )
     self.assertGreater(scores[0]					,  0     )
     self.assertEqual(indices[0]					,  1     )





@require_faiss
def   a_    ( lowerCAmelCase_      :			Union[str, Any]	):
   import faiss

   __lowerCAmelCase								=	FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT	)
   index.add_vectors(np.eye(5,	dtype=np.floataa	)	)

   __lowerCAmelCase								=	'index.faiss'
   __lowerCAmelCase								=	F"""mock://{index_name}"""
   index.save(lowerCAmelCase_,	storage_options=mockfs.storage_options	)
   __lowerCAmelCase								=	FaissIndex.load(lowerCAmelCase_,	storage_options=mockfs.storage_options	)

   __lowerCAmelCase								=	np.zeros(5,	dtype=np.floataa	)
   __lowerCAmelCase								=	1
   __lowerCAmelCase	,	__lowerCAmelCase								=	index.search(lowerCAmelCase_	)
   assert scores[0] > 0
   assert indices[0] == 1




@require_elasticsearch
class 				_UpperCAmelCase    (    _UpperCamelCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		Any     )							->						int:
     from elasticsearch import Elasticsearch

     with patch('elasticsearch.Elasticsearch.search'     ) as mocked_search, patch(
         'elasticsearch.client.IndicesClient.create'     ) as mocked_index_create, patch('elasticsearch.helpers.streaming_bulk'     ) as mocked_bulk:
        __lowerCAmelCase								=	Elasticsearch()
        __lowerCAmelCase								=	{'acknowledged': True}
        __lowerCAmelCase								=	ElasticSearchIndex(es_client=lowerCAmelCase_     )
        mocked_bulk.return_value([(True, None)] * 3     )
        index.add_documents(['foo', 'bar', 'foobar']     )

        # single query
        __lowerCAmelCase								=	'foo'
        __lowerCAmelCase								=	{'hits': {'hits': [{'_score': 1, '_id': 0}]}}
        __lowerCAmelCase	,	__lowerCAmelCase								=	index.search(lowerCAmelCase_     )
        self.assertEqual(scores[0]					,  1     )
        self.assertEqual(indices[0]					,  0     )

        # single query with timeout
        __lowerCAmelCase								=	'foo'
        __lowerCAmelCase								=	{'hits': {'hits': [{'_score': 1, '_id': 0}]}}
        __lowerCAmelCase	,	__lowerCAmelCase								=	index.search(lowerCAmelCase_					,  request_timeout=3_0     )
        self.assertEqual(scores[0]					,  1     )
        self.assertEqual(indices[0]					,  0     )

        # batched queries
        __lowerCAmelCase								=	['foo', 'bar', 'foobar']
        __lowerCAmelCase								=	{'hits': {'hits': [{'_score': 1, '_id': 1}]}}
        __lowerCAmelCase	,	__lowerCAmelCase								=	index.search_batch(lowerCAmelCase_     )
        __lowerCAmelCase								=	[scores[0] for scores in total_scores]
        __lowerCAmelCase								=	[indices[0] for indices in total_indices]
        self.assertGreater(np.min(lowerCAmelCase_     )					,  0     )
        self.assertListEqual([1, 1, 1]					,  lowerCAmelCase_     )

        # batched queries with timeout
        __lowerCAmelCase								=	['foo', 'bar', 'foobar']
        __lowerCAmelCase								=	{'hits': {'hits': [{'_score': 1, '_id': 1}]}}
        __lowerCAmelCase	,	__lowerCAmelCase								=	index.search_batch(lowerCAmelCase_					,  request_timeout=3_0     )
        __lowerCAmelCase								=	[scores[0] for scores in total_scores]
        __lowerCAmelCase								=	[indices[0] for indices in total_indices]
        self.assertGreater(np.min(lowerCAmelCase_     )					,  0     )
        self.assertListEqual([1, 1, 1]					,  lowerCAmelCase_     )

