
import importlib
import inspect
import json
import os
import re
import shutil
import sys
from pathlib import Path
from typing import Dict, Optional, Union
from urllib import request

from huggingface_hub import HfFolder, cached_download, hf_hub_download, model_info
from packaging import version

from .. import __version__
from . import DIFFUSERS_DYNAMIC_MODULE_NAME, HF_MODULES_CACHE, logging


A     :	Optional[Any]    = (
    "https://raw.githubusercontent.com/huggingface/diffusers/{revision}/examples/community/{pipeline}.py"
)


A     :	str    = logging.get_logger(__name__)  # pylint: disable=invalid-name


def     __lowerCAmelCase			(			)       ->  Tuple:
				__a										=    '''https://pypi.org/pypi/diffusers/json'''
				__a										=    json.loads(request.urlopen(lowerCAmelCase__  ).read()  )['''releases'''].keys()
				return sorted(lowerCAmelCase__						,    key=lambda a__  : version.Version(lowerCAmelCase__  )  )


def     __lowerCAmelCase			(			)       ->  Dict:
				# This function has already been executed if HF_MODULES_CACHE already is in the Python path.
				if HF_MODULES_CACHE in sys.path:
								return

				sys.path.append(lowerCAmelCase__  )
				os.makedirs(lowerCAmelCase__						,    exist_ok=lowerCAmelCase__  )
				__a										=    Path(lowerCAmelCase__  ) / '''__init__.py'''
				if not init_path.exists():
								init_path.touch()


def     __lowerCAmelCase			(			a__  )       ->  Optional[Any]:
				init_hf_modules()
				__a										=    Path(lowerCAmelCase__  ) / name
				# If the parent module does not exist yet, recursively create it.
				if not dynamic_module_path.parent.exists():
								create_dynamic_module(dynamic_module_path.parent  )
				os.makedirs(lowerCAmelCase__						,    exist_ok=lowerCAmelCase__  )
				__a										=    dynamic_module_path / '''__init__.py'''
				if not init_path.exists():
								init_path.touch()


def     __lowerCAmelCase			(			a__  )       ->  List[Any]:
				with open(lowerCAmelCase__						,    '''r'''						,    encoding='''utf-8'''  ) as f:
								__a										=    f.read()

				# Imports of the form `import .xxx`
				__a										=    re.findall('''^\s*import\s+\.(\S+)\s*$'''						,    lowerCAmelCase__						,    flags=re.MULTILINE  )
				# Imports of the form `from .xxx import yyy`
				relative_imports += re.findall('''^\s*from\s+\.(\S+)\s+import'''						,    lowerCAmelCase__						,    flags=re.MULTILINE  )
				# Unique-ify
				return list(set(lowerCAmelCase__  )  )


def     __lowerCAmelCase			(			a__  )       ->  str:
				__a										=    False
				__a										=    [module_file]
				__a										=    []

				# Let's recurse through all relative imports
				while not no_change:
								__a										=    []
								for f in files_to_check:
												new_imports.extend(get_relative_imports(lowerCAmelCase__  )  )

								__a										=    Path(lowerCAmelCase__  ).parent
								__a										=    [str(module_path / m  ) for m in new_imports]
								__a										=    [f for f in new_import_files if f not in all_relative_imports]
								__a										=    [F"""{f}.py""" for f in new_import_files]

								__a										=    len(lowerCAmelCase__  ) == 0
								all_relative_imports.extend(lowerCAmelCase__  )

				return all_relative_imports


def     __lowerCAmelCase			(			a__  )       ->  str:
				with open(lowerCAmelCase__						,    '''r'''						,    encoding='''utf-8'''  ) as f:
								__a										=    f.read()

				# Imports of the form `import xxx`
				__a										=    re.findall('''^\s*import\s+(\S+)\s*$'''						,    lowerCAmelCase__						,    flags=re.MULTILINE  )
				# Imports of the form `from xxx import yyy`
				imports += re.findall('''^\s*from\s+(\S+)\s+import'''						,    lowerCAmelCase__						,    flags=re.MULTILINE  )
				# Only keep the top-level module
				__a										=    [imp.split('''.'''  )[0] for imp in imports if not imp.startswith('''.'''  )]

				# Unique-ify and test we got them all
				__a										=    list(set(lowerCAmelCase__  )  )
				__a										=    []
				for imp in imports:
								try:
												importlib.import_module(lowerCAmelCase__  )
								except ImportError:
												missing_packages.append(lowerCAmelCase__  )

				if len(lowerCAmelCase__  ) > 0:
								raise ImportError(
								    '''This modeling file requires the following packages that were not found in your environment: '''
								    F"""{', '.join(lowerCAmelCase__  )}. Run `pip install {' '.join(lowerCAmelCase__  )}`"""  )

				return get_relative_imports(lowerCAmelCase__  )


def     __lowerCAmelCase			(			a__						,    a__  )       ->  Optional[int]:
				__a										=    module_path.replace(os.path.sep						,    '''.'''  )
				__a										=    importlib.import_module(lowerCAmelCase__  )

				if class_name is None:
								return find_pipeline_class(lowerCAmelCase__  )
				return getattr(lowerCAmelCase__						,    lowerCAmelCase__  )


def     __lowerCAmelCase			(			a__  )       ->  Tuple:
				from ..pipelines import DiffusionPipeline

				__a										=    dict(inspect.getmembers(lowerCAmelCase__						,    inspect.isclass  )  )

				__a										=    None
				for cls_name, cls in cls_members.items():
								if (
								    cls_name != DiffusionPipeline.__name__
								    and issubclass(cls						,    lowerCAmelCase__  )
								    and cls.__module__.split('''.'''  )[0] != "diffusers"
								):
												if pipeline_class is not None:
																raise ValueError(
																    F"""Multiple classes that inherit from {DiffusionPipeline.__name__} have been found:"""
																    F""" {pipeline_class.__name__}, and {cls_name}. Please make sure to define only one in"""
																    F""" {loaded_module}."""  )
												__a										=    cls

				return pipeline_class


def     __lowerCAmelCase			(			a__						,    a__						,    a__ = None						,    a__ = False						,    a__ = False						,    a__ = None						,    a__ = None						,    a__ = None						,    a__ = False						,    )       ->  List[str]:
				__a										=    str(lowerCAmelCase__  )

				__a										=    os.path.join(lowerCAmelCase__						,    lowerCAmelCase__  )

				if os.path.isfile(lowerCAmelCase__  ):
								__a										=    module_file_or_url
								__a										=    '''local'''
				elif pretrained_model_name_or_path.count('''/'''  ) == 0:
								__a										=    get_diffusers_versions()
								# cut ".dev0"
								__a										=    '''v''' + '''.'''.join(__version__.split('''.'''  )[:3]  )

								# retrieve github version that matches
								if revision is None:
												__a										=    latest_version if latest_version[1:] in available_versions else '''main'''
												logger.info(F"""Defaulting to latest_version: {revision}."""  )
								elif revision in available_versions:
												__a										=    F"""v{revision}"""
								elif revision == "main":
												__a										=    revision
								else:
												raise ValueError(
												    F"""`custom_revision`: {revision} does not exist. Please make sure to choose one of"""
												    F""" {', '.join(available_versions + ['main']  )}."""  )

								# community pipeline on GitHub
								__a										=    COMMUNITY_PIPELINES_URL.format(revision=lowerCAmelCase__						,    pipeline=lowerCAmelCase__  )
								try:
												__a										=    cached_download(
												    lowerCAmelCase__						,    cache_dir=lowerCAmelCase__						,    force_download=lowerCAmelCase__						,    proxies=lowerCAmelCase__						,    resume_download=lowerCAmelCase__						,    local_files_only=lowerCAmelCase__						,    use_auth_token=lowerCAmelCase__						,    )
												__a										=    '''git'''
												__a										=    pretrained_model_name_or_path + '''.py'''
								except EnvironmentError:
												logger.error(F"""Could not locate the {module_file} inside {pretrained_model_name_or_path}."""  )
												raise
				else:
								try:
												# Load from URL or cache if already cached
												__a										=    hf_hub_download(
												    lowerCAmelCase__						,    lowerCAmelCase__						,    cache_dir=lowerCAmelCase__						,    force_download=lowerCAmelCase__						,    proxies=lowerCAmelCase__						,    resume_download=lowerCAmelCase__						,    local_files_only=lowerCAmelCase__						,    use_auth_token=lowerCAmelCase__						,    )
												__a										=    os.path.join('''local'''						,    '''--'''.join(pretrained_model_name_or_path.split('''/'''  )  )  )
								except EnvironmentError:
												logger.error(F"""Could not locate the {module_file} inside {pretrained_model_name_or_path}."""  )
												raise

    # Check we have all the requirements in our environment
				__a										=    check_imports(lowerCAmelCase__  )

				# Now we move the module inside our cached dynamic modules.
				__a										=    DIFFUSERS_DYNAMIC_MODULE_NAME + os.path.sep + submodule
				create_dynamic_module(lowerCAmelCase__  )
				__a										=    Path(lowerCAmelCase__  ) / full_submodule
				if submodule == "local" or submodule == "git":
								# We always copy local files (we could hash the file to see if there was a change, and give them the name of
								# that hash, to only copy when there is a modification but it seems overkill for now).
								# The only reason we do the copy is to avoid putting too many folders in sys.path.
								shutil.copy(lowerCAmelCase__						,    submodule_path / module_file  )
								for module_needed in modules_needed:
												__a										=    F"""{module_needed}.py"""
												shutil.copy(os.path.join(lowerCAmelCase__						,    lowerCAmelCase__  )						,    submodule_path / module_needed  )
				else:
								# Get the commit hash
								# TODO: we will get this info in the etag soon, so retrieve it from there and not here.
								if isinstance(lowerCAmelCase__						,    lowerCAmelCase__  ):
												__a										=    use_auth_token
								elif use_auth_token is True:
												__a										=    HfFolder.get_token()
								else:
												__a										=    None

								__a										=    model_info(lowerCAmelCase__						,    revision=lowerCAmelCase__						,    token=lowerCAmelCase__  ).sha

								# The module file will end up being placed in a subfolder with the git hash of the repo. This way we get the
								# benefit of versioning.
								__a										=    submodule_path / commit_hash
								__a										=    full_submodule + os.path.sep + commit_hash
								create_dynamic_module(lowerCAmelCase__  )

								if not (submodule_path / module_file).exists():
												shutil.copy(lowerCAmelCase__						,    submodule_path / module_file  )
								# Make sure we also have every file with relative
								for module_needed in modules_needed:
												if not (submodule_path / module_needed).exists():
																get_cached_module_file(
																    lowerCAmelCase__						,    F"""{module_needed}.py"""						,    cache_dir=lowerCAmelCase__						,    force_download=lowerCAmelCase__						,    resume_download=lowerCAmelCase__						,    proxies=lowerCAmelCase__						,    use_auth_token=lowerCAmelCase__						,    revision=lowerCAmelCase__						,    local_files_only=lowerCAmelCase__						,    )
				return os.path.join(lowerCAmelCase__						,    lowerCAmelCase__  )






def     __lowerCAmelCase			(			a__						,    a__						,    a__ = None						,    a__ = None						,    a__ = False						,    a__ = False						,    a__ = None						,    a__ = None						,    a__ = None						,    a__ = False						,    **a__						,    )       ->  Tuple:
				__a										=    get_cached_module_file(
				    lowerCAmelCase__						,    lowerCAmelCase__						,    cache_dir=lowerCAmelCase__						,    force_download=lowerCAmelCase__						,    resume_download=lowerCAmelCase__						,    proxies=lowerCAmelCase__						,    use_auth_token=lowerCAmelCase__						,    revision=lowerCAmelCase__						,    local_files_only=lowerCAmelCase__						,    )
				return get_class_in_module(lowerCAmelCase__						,    final_module.replace('''.py'''						,    ''''''  )  )

import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


lowercase__		:Optional[int]   =    pytest.mark.integration







@require_faiss
class 				lowercase		(					SCREAMING_SNAKE_CASE__		):




							def  A__      (				self):
												lowercase							=						Dataset.from_dict({'''filename''': ['''my_name-train''' + '''_''' + str(A__) for x in np.arange(3_0).tolist()]})
												return dset




							def  A__      (				self):
												import faiss

												lowercase							=						self._create_dummy_dataset()
												lowercase							=						dset.map(
												    lambda A__   ,A__: {"vecs": i * np.ones(5   ,dtype=np.floataa)}   ,with_indices=A__   ,keep_in_memory=A__)
												lowercase							=						dset.add_faiss_index('''vecs'''   ,batch_size=1_0_0   ,metric_type=faiss.METRIC_INNER_PRODUCT)
												lowercase	,						lowercase							=						dset.get_nearest_examples('''vecs'''   ,np.ones(5   ,dtype=np.floataa))
												self.assertEqual(examples['''filename'''][0]   ,'''my_name-train_29''')
												dset.drop_index('''vecs''')




							def  A__      (				self):
												import faiss

												lowercase							=						self._create_dummy_dataset()
												dset.add_faiss_index_from_external_arrays(
												    external_arrays=np.ones((3_0, 5)) * np.arange(3_0).reshape(-1   ,1)   ,index_name='''vecs'''   ,batch_size=1_0_0   ,metric_type=faiss.METRIC_INNER_PRODUCT   ,)
												lowercase	,						lowercase							=						dset.get_nearest_examples('''vecs'''   ,np.ones(5   ,dtype=np.floataa))
												self.assertEqual(examples['''filename'''][0]   ,'''my_name-train_29''')




							def  A__      (				self):
												import faiss

												lowercase							=						self._create_dummy_dataset()
												dset.add_faiss_index_from_external_arrays(
												    external_arrays=np.ones((3_0, 5)) * np.arange(3_0).reshape(-1   ,1)   ,index_name='''vecs'''   ,metric_type=faiss.METRIC_INNER_PRODUCT   ,)

												# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
												# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
												# see https://bugs.python.org/issue14243 and
												# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
												with tempfile.NamedTemporaryFile(delete=A__) as tmp_file:
																	dset.save_faiss_index('''vecs'''   ,tmp_file.name)
																	dset.load_faiss_index('''vecs2'''   ,tmp_file.name)
												os.unlink(tmp_file.name)

												lowercase	,						lowercase							=						dset.get_nearest_examples('''vecs2'''   ,np.ones(5   ,dtype=np.floataa))
												self.assertEqual(examples['''filename'''][0]   ,'''my_name-train_29''')




							def  A__      (				self):
												lowercase							=						self._create_dummy_dataset()
												dset.add_faiss_index_from_external_arrays(
												    external_arrays=np.ones((3_0, 5)) * np.arange(3_0).reshape(-1   ,1)   ,index_name='''vecs''')
												dset.drop_index('''vecs''')
												self.assertRaises(A__   ,partial(dset.get_nearest_examples   ,'''vecs2'''   ,np.ones(5   ,dtype=np.floataa)))





							def  A__      (				self):
												from elasticsearch import Elasticsearch

												lowercase							=						self._create_dummy_dataset()
												with patch('''elasticsearch.Elasticsearch.search''') as mocked_search, patch(
												    '''elasticsearch.client.IndicesClient.create''') as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk''') as mocked_bulk:
																	lowercase							=						{'''acknowledged''': True}
																	mocked_bulk.return_value([(True, None)] * 3_0)
																	lowercase							=						{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 2_9}]}}
																	lowercase							=						Elasticsearch()

																	dset.add_elasticsearch_index('''filename'''   ,es_client=A__)
																	lowercase	,						lowercase							=						dset.get_nearest_examples('''filename'''   ,'''my_name-train_29''')
																	self.assertEqual(examples['''filename'''][0]   ,'''my_name-train_29''')







@require_faiss
class 				lowercase		(					SCREAMING_SNAKE_CASE__		):




							def  A__      (				self):
												import faiss

												lowercase							=						FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT)

												# add vectors
												index.add_vectors(np.eye(5   ,dtype=np.floataa))
												self.assertIsNotNone(index.faiss_index)
												self.assertEqual(index.faiss_index.ntotal   ,5)
												index.add_vectors(np.zeros((5, 5)   ,dtype=np.floataa))
												self.assertEqual(index.faiss_index.ntotal   ,1_0)

												# single query
												lowercase							=						np.zeros(5   ,dtype=np.floataa)
												lowercase							=						1
												lowercase	,						lowercase							=						index.search(A__)
												self.assertRaises(A__   ,index.search   ,query.reshape(-1   ,1))
												self.assertGreater(scores[0]   ,0)
												self.assertEqual(indices[0]   ,1)

												# batched queries
												lowercase							=						np.eye(5   ,dtype=np.floataa)[::-1]
												lowercase	,						lowercase							=						index.search_batch(A__)
												self.assertRaises(A__   ,index.search_batch   ,queries[0])
												lowercase							=						[scores[0] for scores in total_scores]
												lowercase							=						[indices[0] for indices in total_indices]
												self.assertGreater(np.min(A__)   ,0)
												self.assertListEqual([4, 3, 2, 1, 0]   ,A__)




							def  A__      (				self):
												import faiss

												lowercase							=						FaissIndex(string_factory='''Flat''')
												index.add_vectors(np.eye(5   ,dtype=np.floataa))
												self.assertIsInstance(index.faiss_index   ,faiss.IndexFlat)
												lowercase							=						FaissIndex(string_factory='''LSH''')
												index.add_vectors(np.eye(5   ,dtype=np.floataa))
												self.assertIsInstance(index.faiss_index   ,faiss.IndexLSH)
												with self.assertRaises(A__):
																	lowercase							=						FaissIndex(string_factory='''Flat'''   ,custom_index=faiss.IndexFlat(5))




							def  A__      (				self):
												import faiss

												lowercase							=						faiss.IndexFlat(5)
												lowercase							=						FaissIndex(custom_index=A__)
												index.add_vectors(np.eye(5   ,dtype=np.floataa))
												self.assertIsInstance(index.faiss_index   ,faiss.IndexFlat)





							def  A__      (				self):
												import faiss

												lowercase							=						FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT)
												index.add_vectors(np.eye(5   ,dtype=np.floataa))

												# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
												# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
												# see https://bugs.python.org/issue14243 and
												# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
												with tempfile.NamedTemporaryFile(delete=A__) as tmp_file:
																	index.save(tmp_file.name)
																	lowercase							=						FaissIndex.load(tmp_file.name)
												os.unlink(tmp_file.name)

												lowercase							=						np.zeros(5   ,dtype=np.floataa)
												lowercase							=						1
												lowercase	,						lowercase							=						index.search(A__)
												self.assertGreater(scores[0]   ,0)
												self.assertEqual(indices[0]   ,1)



@require_faiss
def        UpperCamelCase    (							lowerCAmelCase__		):
					'''simple docstring'''
					import faiss

					lowercase							=						FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT		)
					index.add_vectors(np.eye(5 ,			dtype=np.floataa		)		)

					lowercase							=						'''index.faiss'''
					lowercase							=						f'mock://{index_name}'
					index.save(lowerCAmelCase__ ,			storage_options=mockfs.storage_options		)
					lowercase							=						FaissIndex.load(lowerCAmelCase__ ,			storage_options=mockfs.storage_options		)

					lowercase							=						np.zeros(5 ,			dtype=np.floataa		)
					lowercase							=						1
					lowercase	,						lowercase							=						index.search(lowerCAmelCase__		)
					assert scores[0] > 0
					assert indices[0] == 1







@require_elasticsearch
class 				lowercase		(					SCREAMING_SNAKE_CASE__		):




							def  A__      (				self):
												from elasticsearch import Elasticsearch

												with patch('''elasticsearch.Elasticsearch.search''') as mocked_search, patch(
												    '''elasticsearch.client.IndicesClient.create''') as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk''') as mocked_bulk:
																	lowercase							=						Elasticsearch()
																	lowercase							=						{'''acknowledged''': True}
																	lowercase							=						ElasticSearchIndex(es_client=A__)
																	mocked_bulk.return_value([(True, None)] * 3)
																	index.add_documents(['''foo''', '''bar''', '''foobar'''])

																	# single query
																	lowercase							=						'''foo'''
																	lowercase							=						{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
																	lowercase	,						lowercase							=						index.search(A__)
																	self.assertEqual(scores[0]   ,1)
																	self.assertEqual(indices[0]   ,0)

																	# single query with timeout
																	lowercase							=						'''foo'''
																	lowercase							=						{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
																	lowercase	,						lowercase							=						index.search(A__   ,request_timeout=3_0)
																	self.assertEqual(scores[0]   ,1)
																	self.assertEqual(indices[0]   ,0)

																	# batched queries
																	lowercase							=						['''foo''', '''bar''', '''foobar''']
																	lowercase							=						{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
																	lowercase	,						lowercase							=						index.search_batch(A__)
																	lowercase							=						[scores[0] for scores in total_scores]
																	lowercase							=						[indices[0] for indices in total_indices]
																	self.assertGreater(np.min(A__)   ,0)
																	self.assertListEqual([1, 1, 1]   ,A__)

																	# batched queries with timeout
																	lowercase							=						['''foo''', '''bar''', '''foobar''']
																	lowercase							=						{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
																	lowercase	,						lowercase							=						index.search_batch(A__   ,request_timeout=3_0)
																	lowercase							=						[scores[0] for scores in total_scores]
																	lowercase							=						[indices[0] for indices in total_indices]
																	self.assertGreater(np.min(A__)   ,0)
																	self.assertListEqual([1, 1, 1]   ,A__)




