



import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEmbeddings,
    BertLayer,
    BertPooler,
    BertPreTrainedModel,
)


def 			UpperCamelCase(  __UpperCamelCase							:		Optional[Any] ):
		lowerCAmelCase_			:     str									= torch.exp(__UpperCamelCase )
		lowerCAmelCase_			:     Any									= torch.sum(__UpperCamelCase     ,dim=1 )  # sum of exp(x_i)
		lowerCAmelCase_			:     int									= torch.sum(x * exp_x     ,dim=1 )  # sum of x_i * exp(x_i)
		return torch.log(__UpperCamelCase ) - B / A



class 				__snake_case						(     nn.Module						):




							def __init__(       self     :       str				,				A_     :       Optional[Any]):
									super().__init__()
									lowerCAmelCase_			:     Union[str, Any]									= config.output_attentions
									lowerCAmelCase_			:     Tuple									= config.output_hidden_states
									lowerCAmelCase_			:     Optional[Any]									= nn.ModuleList([BertLayer(A_) for _ in range(config.num_hidden_layers)])
									lowerCAmelCase_			:     int									= nn.ModuleList([BertHighway(A_) for _ in range(config.num_hidden_layers)])

									lowerCAmelCase_			:     str									= [-1 for _ in range(config.num_hidden_layers)]




							def    UpperCAmelCase__			(       self     :       Optional[Any]				,				A_     :       Dict):
									if (type(A_) is float) or (type(A_) is int):
											for i in range(len(self.early_exit_entropy)):
													lowerCAmelCase_			:     Optional[Any]									= x
									else:
											lowerCAmelCase_			:     Dict									= x




							def    UpperCAmelCase__			(       self     :       Tuple				,				A_     :       int):
									lowerCAmelCase_			:     Dict									= pooler.state_dict()
									for highway in self.highway:
											for name, param in highway.pooler.state_dict().items():
													param.copy_(loaded_model[name])







							def    UpperCAmelCase__			(       self     :       int				,				A_     :       Tuple				,				A_     :       Tuple=None				,				A_     :       int=None				,				A_     :       str=None				,				A_     :       Dict=None				,				):
									lowerCAmelCase_			:     Optional[int]									= ()
									lowerCAmelCase_			:     str									= ()
									lowerCAmelCase_			:     Optional[Any]									= ()
									for i, layer_module in enumerate(self.layer):
											if self.output_hidden_states:
													lowerCAmelCase_			:     Dict									= all_hidden_states + (hidden_states,)

											lowerCAmelCase_			:     Dict									= layer_module(
											    A_				,				A_				,				head_mask[i]				,				A_				,				A_)
											lowerCAmelCase_			:     str									= layer_outputs[0]

											if self.output_attentions:
													lowerCAmelCase_			:     Optional[int]									= all_attentions + (layer_outputs[1],)

											lowerCAmelCase_			:     List[Any]									= (hidden_states,)
											if self.output_hidden_states:
													lowerCAmelCase_			:     Tuple									= current_outputs + (all_hidden_states,)
											if self.output_attentions:
													lowerCAmelCase_			:     Any									= current_outputs + (all_attentions,)

											lowerCAmelCase_			:     str									= self.highway[i](A_)
											# logits, pooled_output

											if not self.training:
													lowerCAmelCase_			:     List[str]									= highway_exit[0]
													lowerCAmelCase_			:     str									= entropy(A_)
													lowerCAmelCase_			:     str									= highway_exit + (highway_entropy,)  # logits, hidden_states(?), entropy
													lowerCAmelCase_			:     str									= all_highway_exits + (highway_exit,)

													if highway_entropy < self.early_exit_entropy[i]:
															lowerCAmelCase_			:     str									= (highway_logits,) + current_outputs[1:] + (all_highway_exits,)
															raise HighwayException(A_				,				i + 1)
											else:
													lowerCAmelCase_			:     List[Any]									= all_highway_exits + (highway_exit,)

        # Add last layer
									if self.output_hidden_states:
											lowerCAmelCase_			:     Dict									= all_hidden_states + (hidden_states,)

									lowerCAmelCase_			:     str									= (hidden_states,)
									if self.output_hidden_states:
											lowerCAmelCase_			:     str									= outputs + (all_hidden_states,)
									if self.output_attentions:
											lowerCAmelCase_			:     Optional[Any]									= outputs + (all_attentions,)

									lowerCAmelCase_			:     Dict									= outputs + (all_highway_exits,)
									return outputs  # last-layer hidden state, (all hidden states), (all attentions), all highway exits





@add_start_docstrings(
    '''The Bert Model transformer with early exiting (DeeBERT). '''						,UpperCamelCase_						,)
class 				__snake_case						(     UpperCamelCase_						):




							def __init__(       self     :       str				,				A_     :       Optional[int]):
									super().__init__(A_)
									lowerCAmelCase_			:     List[str]									= config

									lowerCAmelCase_			:     List[Any]									= BertEmbeddings(A_)
									lowerCAmelCase_			:     Union[str, Any]									= DeeBertEncoder(A_)
									lowerCAmelCase_			:     Any									= BertPooler(A_)

									self.init_weights()




							def    UpperCAmelCase__			(       self     :       List[Any]):
									self.encoder.init_highway_pooler(self.pooler)




							def    UpperCAmelCase__			(       self     :       Union[str, Any]):
									return self.embeddings.word_embeddings




							def    UpperCAmelCase__			(       self     :       List[str]				,				A_     :       List[Any]):
									lowerCAmelCase_			:     Dict									= value




							def    UpperCAmelCase__			(       self     :       Tuple				,				A_     :       int):
									for layer, heads in heads_to_prune.items():
											self.encoder.layer[layer].attention.prune_heads(A_)







							@add_start_docstrings_to_model_forward(A_)
							def    UpperCAmelCase__			(       self     :       Tuple				,				A_     :       Union[str, Any]=None				,				A_     :       str=None				,				A_     :       str=None				,				A_     :       str=None				,				A_     :       List[str]=None				,				A_     :       Optional[int]=None				,				A_     :       Union[str, Any]=None				,				A_     :       Any=None				,				):
									if input_ids is not None and inputs_embeds is not None:
											raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time''')
									elif input_ids is not None:
											lowerCAmelCase_			:     List[Any]									= input_ids.size()
									elif inputs_embeds is not None:
											lowerCAmelCase_			:     Dict									= inputs_embeds.size()[:-1]
									else:
											raise ValueError('''You have to specify either input_ids or inputs_embeds''')

									lowerCAmelCase_			:     Any									= input_ids.device if input_ids is not None else inputs_embeds.device

									if attention_mask is None:
											lowerCAmelCase_			:     Tuple									= torch.ones(A_				,				device=A_)
									if encoder_attention_mask is None:
											lowerCAmelCase_			:     str									= torch.ones(A_				,				device=A_)
									if token_type_ids is None:
											lowerCAmelCase_			:     str									= torch.zeros(A_				,				dtype=torch.long				,				device=A_)

									# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
									# ourselves in which case we just need to make it broadcastable to all heads.
									lowerCAmelCase_			:     torch.Tensor									= self.get_extended_attention_mask(A_				,				A_				,				A_)

									# If a 2D ou 3D attention mask is provided for the cross-attention
									# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
									if encoder_attention_mask.dim() == 3:
											lowerCAmelCase_			:     List[str]									= encoder_attention_mask[:, None, :, :]
									if encoder_attention_mask.dim() == 2:
											lowerCAmelCase_			:     Dict									= encoder_attention_mask[:, None, None, :]

									lowerCAmelCase_			:     Union[str, Any]									= encoder_extended_attention_mask.to(
									    dtype=next(self.parameters()).dtype)  # fp16 compatibility
									lowerCAmelCase_			:     str									= (1.0 - encoder_extended_attention_mask) * -1_0000.0

									# Prepare head mask if needed
									# 1.0 in head_mask indicate we keep the head
									# attention_probs has shape bsz x n_heads x N x N
									# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
									# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
									lowerCAmelCase_			:     int									= self.get_head_mask(A_				,				self.config.num_hidden_layers)

									lowerCAmelCase_			:     str									= self.embeddings(
									    input_ids=A_				,				position_ids=A_				,				token_type_ids=A_				,				inputs_embeds=A_)
									lowerCAmelCase_			:     Union[str, Any]									= self.encoder(
									    A_				,				attention_mask=A_				,				head_mask=A_				,				encoder_hidden_states=A_				,				encoder_attention_mask=A_				,				)
									lowerCAmelCase_			:     int									= encoder_outputs[0]
									lowerCAmelCase_			:     Dict									= self.pooler(A_)

									lowerCAmelCase_			:     Tuple									= (
									    sequence_output,
									    pooled_output,
									) + encoder_outputs[
									    1:
									]  # add hidden_states and attentions if they are here
									return outputs  # sequence_output, pooled_output, (hidden_states), (attentions), highway exits





class 				__snake_case						(     UpperCamelCase_						):




							def __init__(       self     :       List[Any]				,				A_     :       Optional[int]				,				A_     :       Optional[int]):
									lowerCAmelCase_			:     Tuple									= message
									lowerCAmelCase_			:     List[str]									= exit_layer  # start from 1!





class 				__snake_case						(     nn.Module						):




							def __init__(       self     :       List[str]				,				A_     :       Any):
									super().__init__()
									lowerCAmelCase_			:     int									= BertPooler(A_)
									lowerCAmelCase_			:     Any									= nn.Dropout(config.hidden_dropout_prob)
									lowerCAmelCase_			:     str									= nn.Linear(config.hidden_size				,				config.num_labels)







							def    UpperCAmelCase__			(       self     :       str				,				A_     :       str):
									# Pooler
									lowerCAmelCase_			:     Optional[Any]									= encoder_outputs[0]
									lowerCAmelCase_			:     str									= self.pooler(A_)
									# "return" pooler_output

									# BertModel
									lowerCAmelCase_			:     Optional[int]									= (pooler_input, pooler_output) + encoder_outputs[1:]
									# "return" bmodel_output

									# Dropout and classification
									lowerCAmelCase_			:     int									= bmodel_output[1]

									lowerCAmelCase_			:     Tuple									= self.dropout(A_)
									lowerCAmelCase_			:     Optional[Any]									= self.classifier(A_)

									return logits, pooled_output





@add_start_docstrings(
    '''Bert Model (with early exiting - DeeBERT) with a classifier on top,
    also takes care of multi-layer training. '''						,UpperCamelCase_						,)
class 				__snake_case						(     UpperCamelCase_						):




							def __init__(       self     :       Dict				,				A_     :       str):
									super().__init__(A_)
									lowerCAmelCase_			:     Tuple									= config.num_labels
									lowerCAmelCase_			:     Union[str, Any]									= config.num_hidden_layers

									lowerCAmelCase_			:     int									= DeeBertModel(A_)
									lowerCAmelCase_			:     List[Any]									= nn.Dropout(config.hidden_dropout_prob)
									lowerCAmelCase_			:     Union[str, Any]									= nn.Linear(config.hidden_size				,				self.config.num_labels)

									self.init_weights()







							@add_start_docstrings_to_model_forward(A_)
							def    UpperCAmelCase__			(       self     :       Union[str, Any]				,				A_     :       List[str]=None				,				A_     :       Any=None				,				A_     :       Any=None				,				A_     :       List[Any]=None				,				A_     :       Tuple=None				,				A_     :       List[Any]=None				,				A_     :       str=None				,				A_     :       str=-1				,				A_     :       List[Any]=False				,				):
									lowerCAmelCase_			:     Any									= self.num_layers
									try:
											lowerCAmelCase_			:     str									= self.bert(
											    A_				,				attention_mask=A_				,				token_type_ids=A_				,				position_ids=A_				,				head_mask=A_				,				inputs_embeds=A_				,				)
											# sequence_output, pooled_output, (hidden_states), (attentions), highway exits

											lowerCAmelCase_			:     Union[str, Any]									= outputs[1]

											lowerCAmelCase_			:     List[Any]									= self.dropout(A_)
											lowerCAmelCase_			:     Any									= self.classifier(A_)
											lowerCAmelCase_			:     List[str]									= (logits,) + outputs[2:]  # add hidden states and attention if they are here
									except HighwayException as e:
											lowerCAmelCase_			:     Optional[int]									= e.message
											lowerCAmelCase_			:     List[str]									= e.exit_layer
											lowerCAmelCase_			:     List[str]									= outputs[0]

									if not self.training:
											lowerCAmelCase_			:     List[str]									= entropy(A_)
											lowerCAmelCase_			:     Optional[int]									= []
											lowerCAmelCase_			:     int									= []
									if labels is not None:
											if self.num_labels == 1:
													#  We are doing regression
													lowerCAmelCase_			:     Dict									= MSELoss()
													lowerCAmelCase_			:     Any									= loss_fct(logits.view(-1)				,				labels.view(-1))
											else:
													lowerCAmelCase_			:     Optional[int]									= CrossEntropyLoss()
													lowerCAmelCase_			:     Tuple									= loss_fct(logits.view(-1				,				self.num_labels)				,				labels.view(-1))

											# work with highway exits
											lowerCAmelCase_			:     Any									= []
											for highway_exit in outputs[-1]:
													lowerCAmelCase_			:     int									= highway_exit[0]
													if not self.training:
															highway_logits_all.append(A_)
															highway_entropy.append(highway_exit[2])
													if self.num_labels == 1:
															#  We are doing regression
															lowerCAmelCase_			:     Dict									= MSELoss()
															lowerCAmelCase_			:     Optional[Any]									= loss_fct(highway_logits.view(-1)				,				labels.view(-1))
													else:
															lowerCAmelCase_			:     List[str]									= CrossEntropyLoss()
															lowerCAmelCase_			:     Union[str, Any]									= loss_fct(highway_logits.view(-1				,				self.num_labels)				,				labels.view(-1))
													highway_losses.append(A_)

											if train_highway:
													lowerCAmelCase_			:     List[Any]									= (sum(highway_losses[:-1]),) + outputs
													# exclude the final highway, of course
											else:
													lowerCAmelCase_			:     int									= (loss,) + outputs
									if not self.training:
											lowerCAmelCase_			:     Tuple									= outputs + ((original_entropy, highway_entropy), exit_layer)
											if output_layer >= 0:
													lowerCAmelCase_			:     Tuple									= (
													    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
													)  # use the highway of the last layer

									return outputs  # (loss), logits, (hidden_states), (attentions), (highway_exits)










# Lint as: python3
import os
import re
import urllib.parse
from pathlib import Path
from typing import Callable, List, Optional, Union
from zipfile import ZipFile

from ..utils.file_utils import cached_path, hf_github_url
from ..utils.logging import get_logger
from ..utils.version import Version


A__  :  Tuple								=					get_logger(__name__)



class 				__snake_case						:
							_a			      = '''dummy_data'''
							_a			      = '''datasets'''
							_a			      = False




							def __init__(       self     :       Optional[Any]				,				A_     :       str				,				A_     :       str				,				A_     :       Union[Version, str]				,				A_     :       Optional[str] = None				,				A_     :       bool = False				,				A_     :       bool = True				,				A_     :       Optional[List[Callable]] = None				,				):
									lowerCAmelCase_			:     List[Any]									= 0
									lowerCAmelCase_			:     Any									= dataset_name
									lowerCAmelCase_			:     Union[str, Any]									= cache_dir
									lowerCAmelCase_			:     List[Any]									= use_local_dummy_data
									lowerCAmelCase_			:     Optional[Any]									= config
									# download_callbacks take a single url as input
									lowerCAmelCase_			:     List[Callable]									= download_callbacks or []
									# if False, it doesn't load existing files and it returns the paths of the dummy files relative
									# to the dummy_data zip file root
									lowerCAmelCase_			:     Tuple									= load_existing_dummy_data

									# TODO(PVP, QL) might need to make this more general
									lowerCAmelCase_			:     int									= str(A_)
									# to be downloaded
									lowerCAmelCase_			:     Dict									= None
									lowerCAmelCase_			:     Optional[int]									= None




							@property
							def    UpperCAmelCase__			(       self     :       List[str]):
									if self._dummy_file is None:
											lowerCAmelCase_			:     int									= self.download_dummy_data()
									return self._dummy_file




							@property
							def    UpperCAmelCase__			(       self     :       str):
									if self.config is not None:
											# structure is dummy / config_name / version_name
											return os.path.join('''dummy'''				,				self.config.name				,				self.version_name)
									# structure is dummy / version_name
									return os.path.join('''dummy'''				,				self.version_name)




							@property
							def    UpperCAmelCase__			(       self     :       str):
									return os.path.join(self.dummy_data_folder				,				'''dummy_data.zip''')




							def    UpperCAmelCase__			(       self     :       Any):
									lowerCAmelCase_			:     Any									= (
									    self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data
									)

									lowerCAmelCase_			:     Union[str, Any]									= cached_path(
									    A_				,				cache_dir=self.cache_dir				,				extract_compressed_file=A_				,				force_extract=A_)

									return os.path.join(A_				,				self.dummy_file_name)




							@property
							def    UpperCAmelCase__			(       self     :       List[str]):
									return os.path.join(self.datasets_scripts_dir				,				self.dataset_name				,				self.dummy_zip_file)




							@property
							def    UpperCAmelCase__			(       self     :       Optional[int]):
									if self._bucket_url is None:
											lowerCAmelCase_			:     str									= hf_github_url(self.dataset_name				,				self.dummy_zip_file.replace(os.sep				,				'''/'''))
									return self._bucket_url




							@property
							def    UpperCAmelCase__			(       self     :       List[Any]):
									# return full path if its a dir
									if os.path.isdir(self.dummy_file):
											return self.dummy_file
									# else cut off path to file -> example `xsum`.
									return "/".join(self.dummy_file.replace(os.sep				,				'''/''').split('''/''')[:-1])




							def    UpperCAmelCase__			(       self     :       Union[str, Any]				,				A_     :       Dict				,				*A_     :       List[Any]):
									if self.load_existing_dummy_data:
											# dummy data is downloaded and tested
											lowerCAmelCase_			:     Union[str, Any]									= self.dummy_file
									else:
											# dummy data cannot be downloaded and only the path to dummy file is returned
											lowerCAmelCase_			:     Optional[int]									= self.dummy_file_name

									# special case when data_url is a dict
									if isinstance(A_				,				A_):
											return self.create_dummy_data_dict(A_				,				A_)
									elif isinstance(A_				,				(list, tuple)):
											return self.create_dummy_data_list(A_				,				A_)
									else:
											return self.create_dummy_data_single(A_				,				A_)




							def    UpperCAmelCase__			(       self     :       Optional[int]				,				A_     :       Tuple				,				*A_     :       int):
									return self.download_and_extract(A_)




							def    UpperCAmelCase__			(       self     :       Tuple				,				A_     :       List[str]				,				A_     :       Optional[Any]):
									return self.download_and_extract(A_)




							def    UpperCAmelCase__			(       self     :       int				,				A_     :       Optional[int]				,				*A_     :       str				,				**A_     :       List[Any]):
									return path




							def    UpperCAmelCase__			(       self     :       Tuple):
									return {}




							def    UpperCAmelCase__			(       self     :       Optional[Any]				,				A_     :       Union[str, Any]				,				A_     :       List[Any]):
									lowerCAmelCase_			:     Union[str, Any]									= {}
									for key, single_urls in data_url.items():
											for download_callback in self.download_callbacks:
													if isinstance(A_				,				A_):
															for single_url in single_urls:
																	download_callback(A_)
													else:
															lowerCAmelCase_			:     Any									= single_urls
															download_callback(A_)
            # we force the name of each key to be the last file / folder name of the url path
            # if the url has arguments, we need to encode them with urllib.parse.quote_plus
											if isinstance(A_				,				A_):
													lowerCAmelCase_			:     Any									= [os.path.join(A_				,				urllib.parse.quote_plus(Path(A_).name)) for x in single_urls]
											else:
													lowerCAmelCase_			:     Optional[int]									= single_urls
													lowerCAmelCase_			:     List[str]									= os.path.join(A_				,				urllib.parse.quote_plus(Path(A_).name))
											lowerCAmelCase_			:     Dict									= value

									# make sure that values are unique
									if all(isinstance(A_				,				A_) for i in dummy_data_dict.values()) and len(set(dummy_data_dict.values())) < len(
									    dummy_data_dict.values()):
											# append key to value to make its name unique
											lowerCAmelCase_			:     Tuple									= {key: value + key for key, value in dummy_data_dict.items()}

									return dummy_data_dict




							def    UpperCAmelCase__			(       self     :       Dict				,				A_     :       List[str]				,				A_     :       str):
									lowerCAmelCase_			:     Optional[Any]									= []
									# trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one
									lowerCAmelCase_			:     str									= all(bool(re.findall('''[0-9]{3,}-of-[0-9]{3,}'''				,				A_)) for url in data_url)
									lowerCAmelCase_			:     Optional[Any]									= all(
									    url.startswith('''https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed''') for url in data_url)
									if data_url and (is_tf_records or is_pubmed_records):
											lowerCAmelCase_			:     Any									= [data_url[0]] * len(A_)
									for single_url in data_url:
											for download_callback in self.download_callbacks:
													download_callback(A_)
											# we force the name of each key to be the last file / folder name of the url path
											# if the url has arguments, we need to encode them with urllib.parse.quote_plus
											lowerCAmelCase_			:     int									= os.path.join(A_				,				urllib.parse.quote_plus(single_url.split('''/''')[-1]))
											dummy_data_list.append(A_)
									return dummy_data_list




							def    UpperCAmelCase__			(       self     :       List[str]				,				A_     :       Optional[Any]				,				A_     :       Tuple):
									for download_callback in self.download_callbacks:
											download_callback(A_)
									# we force the name of each key to be the last file / folder name of the url path
									# if the url has arguments, we need to encode them with urllib.parse.quote_plus
									lowerCAmelCase_			:     Tuple									= os.path.join(A_				,				urllib.parse.quote_plus(data_url.split('''/''')[-1]))
									if os.path.exists(A_) or not self.load_existing_dummy_data:
											return value
									else:
											# Backward compatibility, maybe deprecate at one point.
											# For many datasets with single url calls to dl_manager.download_and_extract,
											# the dummy_data.zip file is actually the zipped downloaded file
											# while now we expected the dummy_data.zip file to be a directory containing
											# the downloaded file.
											return path_to_dummy_data




							def    UpperCAmelCase__			(       self     :       int):
									pass




							def    UpperCAmelCase__			(       self     :       Optional[int]):
									pass




							def    UpperCAmelCase__			(       self     :       List[str]				,				A_     :       str):
									def _iter_archive_members(A_     :       Any):
											# this preserves the order of the members inside the ZIP archive
											lowerCAmelCase_			:     Optional[int]									= Path(self.dummy_file).parent
											lowerCAmelCase_			:     Optional[int]									= path.relative_to(A_)
											with ZipFile(self.local_path_to_dummy_data) as zip_file:
													lowerCAmelCase_			:     Tuple									= zip_file.namelist()
											for member in members:
													if member.startswith(relative_path.as_posix()):
															yield dummy_parent_path.joinpath(A_)

									lowerCAmelCase_			:     List[Any]									= Path(A_)
									lowerCAmelCase_			:     Optional[int]									= _iter_archive_members(A_) if self.use_local_dummy_data else path.rglob('''*''')
									for file_path in file_paths:
											if file_path.is_file() and not file_path.name.startswith(('''.''', '''__''')):
													yield file_path.relative_to(A_).as_posix(), file_path.open('''rb''')







							def    UpperCAmelCase__			(       self     :       Dict				,				A_     :       Any):
									if not isinstance(A_				,				A_):
											lowerCAmelCase_			:     Dict									= [paths]
									for path in paths:
											if os.path.isfile(A_):
													if os.path.basename(A_).startswith(('''.''', '''__''')):
															return
													yield path
											else:
													for dirpath, dirnames, filenames in os.walk(A_):
															if os.path.basename(A_).startswith(('''.''', '''__''')):
																	continue
															dirnames.sort()
															for filename in sorted(A_):
																	if filename.startswith(('''.''', '''__''')):
																			continue
																	yield os.path.join(A_				,				A_)





