





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


_snake_case  :    Dict						    =  logging.get_logger(__name__)

_snake_case  :    Optional[int]						    =  {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_snake_case  :    str						    =  {
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

_snake_case  :    List[str]						    =  {
    'distilbert-base-uncased': 512,
    'distilbert-base-uncased-distilled-squad': 512,
    'distilbert-base-cased': 512,
    'distilbert-base-cased-distilled-squad': 512,
    'distilbert-base-german-cased': 512,
    'distilbert-base-multilingual-cased': 512,
}


_snake_case  :    int						    =  {
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}



class 				_UpperCAmelCase    (    _UpperCamelCase    ):

  """simple docstring"""


  a_        = VOCAB_FILES_NAMES
  a_        = PRETRAINED_VOCAB_FILES_MAP
  a_        = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_        = PRETRAINED_INIT_CONFIGURATION
  a_        = ["""input_ids""", """attention_mask"""]
  a_        = DistilBertTokenizer






  def __init__(				self :		Union[str, Any]					,  lowerCAmelCase_ :		Union[str, Any]=None					,  lowerCAmelCase_ :		Union[str, Any]=None					,  lowerCAmelCase_ :		int=True					,  lowerCAmelCase_ :		List[Any]="[UNK]"					,  lowerCAmelCase_ :		int="[SEP]"					,  lowerCAmelCase_ :		Optional[int]="[PAD]"					,  lowerCAmelCase_ :		int="[CLS]"					,  lowerCAmelCase_ :		Dict="[MASK]"					,  lowerCAmelCase_ :		List[Any]=True					,  lowerCAmelCase_ :		Tuple=None					,  **lowerCAmelCase_ :		Optional[int]					,  )							->						Optional[int]:
     super().__init__(
         lowerCAmelCase_					,  tokenizer_file=lowerCAmelCase_					,  do_lower_case=lowerCAmelCase_					,  unk_token=lowerCAmelCase_					,  sep_token=lowerCAmelCase_					,  pad_token=lowerCAmelCase_					,  cls_token=lowerCAmelCase_					,  mask_token=lowerCAmelCase_					,  tokenize_chinese_chars=lowerCAmelCase_					,  strip_accents=lowerCAmelCase_					,  **lowerCAmelCase_					,  )

     __lowerCAmelCase								=	json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
     if (
         normalizer_state.get('lowercase'					,  lowerCAmelCase_     ) != do_lower_case
         or normalizer_state.get('strip_accents'					,  lowerCAmelCase_     ) != strip_accents
         or normalizer_state.get('handle_chinese_chars'					,  lowerCAmelCase_     ) != tokenize_chinese_chars
     ):
        __lowerCAmelCase								=	getattr(lowerCAmelCase_					,  normalizer_state.pop('type'     )     )
        __lowerCAmelCase								=	do_lower_case
        __lowerCAmelCase								=	strip_accents
        __lowerCAmelCase								=	tokenize_chinese_chars
        __lowerCAmelCase								=	normalizer_class(**lowerCAmelCase_     )

     __lowerCAmelCase								=	do_lower_case






  def 				lowercase		(				self :		int					,  lowerCAmelCase_ :		Dict					,  lowerCAmelCase_ :		List[Any]=None     )							->						Dict:
     __lowerCAmelCase								=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

     return output






  def 				lowercase		(				self :		str					,  lowerCAmelCase_ :		List[int]					,  lowerCAmelCase_ :		Optional[List[int]] = None     )							->						List[int]:
     __lowerCAmelCase								=	[self.sep_token_id]
     __lowerCAmelCase								=	[self.cls_token_id]
     if token_ids_a is None:
        return len(cls + token_ids_a + sep     ) * [0]
     return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]






  def 				lowercase		(				self :		List[Any]					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		Optional[str] = None     )							->						Tuple[str]:
     __lowerCAmelCase								=	self._tokenizer.model.save(lowerCAmelCase_					,  name=lowerCAmelCase_     )
     return tuple(lowerCAmelCase_     )








import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


_snake_case  :    Dict						    =  pytest.mark.integration



@require_faiss
class 				_UpperCAmelCase    (    _UpperCamelCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		List[Any]     )							->						Optional[Any]:
     __lowerCAmelCase								=	Dataset.from_dict({'filename': ['my_name-train' + '_' + str(lowerCAmelCase_     ) for x in np.arange(3_0     ).tolist()]}     )
     return dset






  def 				lowercase		(				self :		List[str]     )							->						Tuple:
     import faiss

     __lowerCAmelCase								=	self._create_dummy_dataset()
     __lowerCAmelCase								=	dset.map(
         lambda lowerCAmelCase_					,  lowerCAmelCase_     : {"vecs": i * np.ones(5					,  dtype=np.floataa     )}					,  with_indices=lowerCAmelCase_					,  keep_in_memory=lowerCAmelCase_     )
     __lowerCAmelCase								=	dset.add_faiss_index('vecs'					,  batch_size=1_0_0					,  metric_type=faiss.METRIC_INNER_PRODUCT     )
     __lowerCAmelCase	,	__lowerCAmelCase								=	dset.get_nearest_examples('vecs'					,  np.ones(5					,  dtype=np.floataa     )     )
     self.assertEqual(examples['filename'][0]					,  'my_name-train_29'     )
     dset.drop_index('vecs'     )






  def 				lowercase		(				self :		Optional[Any]     )							->						str:
     import faiss

     __lowerCAmelCase								=	self._create_dummy_dataset()
     dset.add_faiss_index_from_external_arrays(
         external_arrays=np.ones((3_0, 5)     ) * np.arange(3_0     ).reshape(-1					,  1     )					,  index_name='vecs'					,  batch_size=1_0_0					,  metric_type=faiss.METRIC_INNER_PRODUCT					,  )
     __lowerCAmelCase	,	__lowerCAmelCase								=	dset.get_nearest_examples('vecs'					,  np.ones(5					,  dtype=np.floataa     )     )
     self.assertEqual(examples['filename'][0]					,  'my_name-train_29'     )






  def 				lowercase		(				self :		int     )							->						Optional[Any]:
     import faiss

     __lowerCAmelCase								=	self._create_dummy_dataset()
     dset.add_faiss_index_from_external_arrays(
         external_arrays=np.ones((3_0, 5)     ) * np.arange(3_0     ).reshape(-1					,  1     )					,  index_name='vecs'					,  metric_type=faiss.METRIC_INNER_PRODUCT					,  )

     # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
     # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
     # see https://bugs.python.org/issue14243 and
     # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
     with tempfile.NamedTemporaryFile(delete=lowerCAmelCase_     ) as tmp_file:
        dset.save_faiss_index('vecs'					,  tmp_file.name     )
        dset.load_faiss_index('vecs2'					,  tmp_file.name     )
     os.unlink(tmp_file.name     )

     __lowerCAmelCase	,	__lowerCAmelCase								=	dset.get_nearest_examples('vecs2'					,  np.ones(5					,  dtype=np.floataa     )     )
     self.assertEqual(examples['filename'][0]					,  'my_name-train_29'     )






  def 				lowercase		(				self :		Union[str, Any]     )							->						List[Any]:
     __lowerCAmelCase								=	self._create_dummy_dataset()
     dset.add_faiss_index_from_external_arrays(
         external_arrays=np.ones((3_0, 5)     ) * np.arange(3_0     ).reshape(-1					,  1     )					,  index_name='vecs'     )
     dset.drop_index('vecs'     )
     self.assertRaises(lowerCAmelCase_					,  partial(dset.get_nearest_examples					,  'vecs2'					,  np.ones(5					,  dtype=np.floataa     )     )     )






  def 				lowercase		(				self :		Union[str, Any]     )							->						Tuple:
     from elasticsearch import Elasticsearch

     __lowerCAmelCase								=	self._create_dummy_dataset()
     with patch('elasticsearch.Elasticsearch.search'     ) as mocked_search, patch(
         'elasticsearch.client.IndicesClient.create'     ) as mocked_index_create, patch('elasticsearch.helpers.streaming_bulk'     ) as mocked_bulk:
        __lowerCAmelCase								=	{'acknowledged': True}
        mocked_bulk.return_value([(True, None)] * 3_0     )
        __lowerCAmelCase								=	{'hits': {'hits': [{'_score': 1, '_id': 2_9}]}}
        __lowerCAmelCase								=	Elasticsearch()

        dset.add_elasticsearch_index('filename'					,  es_client=lowerCAmelCase_     )
        __lowerCAmelCase	,	__lowerCAmelCase								=	dset.get_nearest_examples('filename'					,  'my_name-train_29'     )
        self.assertEqual(examples['filename'][0]					,  'my_name-train_29'     )





@require_faiss
class 				_UpperCAmelCase    (    _UpperCamelCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		str     )							->						int:
     import faiss

     __lowerCAmelCase								=	FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT     )

     # add vectors
     index.add_vectors(np.eye(5					,  dtype=np.floataa     )     )
     self.assertIsNotNone(index.faiss_index     )
     self.assertEqual(index.faiss_index.ntotal					,  5     )
     index.add_vectors(np.zeros((5, 5)					,  dtype=np.floataa     )     )
     self.assertEqual(index.faiss_index.ntotal					,  1_0     )

     # single query
     __lowerCAmelCase								=	np.zeros(5					,  dtype=np.floataa     )
     __lowerCAmelCase								=	1
     __lowerCAmelCase	,	__lowerCAmelCase								=	index.search(lowerCAmelCase_     )
     self.assertRaises(lowerCAmelCase_					,  index.search					,  query.reshape(-1					,  1     )     )
     self.assertGreater(scores[0]					,  0     )
     self.assertEqual(indices[0]					,  1     )

     # batched queries
     __lowerCAmelCase								=	np.eye(5					,  dtype=np.floataa     )[::-1]
     __lowerCAmelCase	,	__lowerCAmelCase								=	index.search_batch(lowerCAmelCase_     )
     self.assertRaises(lowerCAmelCase_					,  index.search_batch					,  queries[0]     )
     __lowerCAmelCase								=	[scores[0] for scores in total_scores]
     __lowerCAmelCase								=	[indices[0] for indices in total_indices]
     self.assertGreater(np.min(lowerCAmelCase_     )					,  0     )
     self.assertListEqual([4, 3, 2, 1, 0]					,  lowerCAmelCase_     )






  def 				lowercase		(				self :		List[Any]     )							->						List[str]:
     import faiss

     __lowerCAmelCase								=	FaissIndex(string_factory='Flat'     )
     index.add_vectors(np.eye(5					,  dtype=np.floataa     )     )
     self.assertIsInstance(index.faiss_index					,  faiss.IndexFlat     )
     __lowerCAmelCase								=	FaissIndex(string_factory='LSH'     )
     index.add_vectors(np.eye(5					,  dtype=np.floataa     )     )
     self.assertIsInstance(index.faiss_index					,  faiss.IndexLSH     )
     with self.assertRaises(lowerCAmelCase_     ):
        __lowerCAmelCase								=	FaissIndex(string_factory='Flat'					,  custom_index=faiss.IndexFlat(5     )     )






  def 				lowercase		(				self :		Union[str, Any]     )							->						Dict:
     import faiss

     __lowerCAmelCase								=	faiss.IndexFlat(5     )
     __lowerCAmelCase								=	FaissIndex(custom_index=lowerCAmelCase_     )
     index.add_vectors(np.eye(5					,  dtype=np.floataa     )     )
     self.assertIsInstance(index.faiss_index					,  faiss.IndexFlat     )






  def 				lowercase		(				self :		str     )							->						Any:
     import faiss

     __lowerCAmelCase								=	FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT     )
     index.add_vectors(np.eye(5					,  dtype=np.floataa     )     )

     # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
     # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
     # see https://bugs.python.org/issue14243 and
     # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
     with tempfile.NamedTemporaryFile(delete=lowerCAmelCase_     ) as tmp_file:
        index.save(tmp_file.name     )
        __lowerCAmelCase								=	FaissIndex.load(tmp_file.name     )
     os.unlink(tmp_file.name     )

     __lowerCAmelCase								=	np.zeros(5					,  dtype=np.floataa     )
     __lowerCAmelCase								=	1
     __lowerCAmelCase	,	__lowerCAmelCase								=	index.search(lowerCAmelCase_     )
     self.assertGreater(scores[0]					,  0     )
     self.assertEqual(indices[0]					,  1     )





@require_faiss
def   a_    ( lowerCAmelCase_      :			Union[str, Any]	):
   import faiss

   __lowerCAmelCase								=	FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT	)
   index.add_vectors(np.eye(5,	dtype=np.floataa	)	)

   __lowerCAmelCase								=	'index.faiss'
   __lowerCAmelCase								=	F"""mock://{index_name}"""
   index.save(lowerCAmelCase_,	storage_options=mockfs.storage_options	)
   __lowerCAmelCase								=	FaissIndex.load(lowerCAmelCase_,	storage_options=mockfs.storage_options	)

   __lowerCAmelCase								=	np.zeros(5,	dtype=np.floataa	)
   __lowerCAmelCase								=	1
   __lowerCAmelCase	,	__lowerCAmelCase								=	index.search(lowerCAmelCase_	)
   assert scores[0] > 0
   assert indices[0] == 1




@require_elasticsearch
class 				_UpperCAmelCase    (    _UpperCamelCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		Any     )							->						int:
     from elasticsearch import Elasticsearch

     with patch('elasticsearch.Elasticsearch.search'     ) as mocked_search, patch(
         'elasticsearch.client.IndicesClient.create'     ) as mocked_index_create, patch('elasticsearch.helpers.streaming_bulk'     ) as mocked_bulk:
        __lowerCAmelCase								=	Elasticsearch()
        __lowerCAmelCase								=	{'acknowledged': True}
        __lowerCAmelCase								=	ElasticSearchIndex(es_client=lowerCAmelCase_     )
        mocked_bulk.return_value([(True, None)] * 3     )
        index.add_documents(['foo', 'bar', 'foobar']     )

        # single query
        __lowerCAmelCase								=	'foo'
        __lowerCAmelCase								=	{'hits': {'hits': [{'_score': 1, '_id': 0}]}}
        __lowerCAmelCase	,	__lowerCAmelCase								=	index.search(lowerCAmelCase_     )
        self.assertEqual(scores[0]					,  1     )
        self.assertEqual(indices[0]					,  0     )

        # single query with timeout
        __lowerCAmelCase								=	'foo'
        __lowerCAmelCase								=	{'hits': {'hits': [{'_score': 1, '_id': 0}]}}
        __lowerCAmelCase	,	__lowerCAmelCase								=	index.search(lowerCAmelCase_					,  request_timeout=3_0     )
        self.assertEqual(scores[0]					,  1     )
        self.assertEqual(indices[0]					,  0     )

        # batched queries
        __lowerCAmelCase								=	['foo', 'bar', 'foobar']
        __lowerCAmelCase								=	{'hits': {'hits': [{'_score': 1, '_id': 1}]}}
        __lowerCAmelCase	,	__lowerCAmelCase								=	index.search_batch(lowerCAmelCase_     )
        __lowerCAmelCase								=	[scores[0] for scores in total_scores]
        __lowerCAmelCase								=	[indices[0] for indices in total_indices]
        self.assertGreater(np.min(lowerCAmelCase_     )					,  0     )
        self.assertListEqual([1, 1, 1]					,  lowerCAmelCase_     )

        # batched queries with timeout
        __lowerCAmelCase								=	['foo', 'bar', 'foobar']
        __lowerCAmelCase								=	{'hits': {'hits': [{'_score': 1, '_id': 1}]}}
        __lowerCAmelCase	,	__lowerCAmelCase								=	index.search_batch(lowerCAmelCase_					,  request_timeout=3_0     )
        __lowerCAmelCase								=	[scores[0] for scores in total_scores]
        __lowerCAmelCase								=	[indices[0] for indices in total_indices]
        self.assertGreater(np.min(lowerCAmelCase_     )					,  0     )
        self.assertListEqual([1, 1, 1]					,  lowerCAmelCase_     )

