

"""simple docstring"""




import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


_lowercase					:     List[str]				 =		re.compile(r'\s+')





def 	lowercase__		(    snake_case_					:Optional[int]  ):
	return {"hash": hashlib.mda(re.sub(UpperCamelCase__       ,						''''''       ,						example['''content''']  ).encode('''utf-8'''  )  ).hexdigest()}





def 	lowercase__		(    snake_case_					:List[Any]  ):
	__UpperCAmelCase							=      [len(UpperCamelCase__  ) for line in example['''content'''].splitlines()]
	return {"line_mean": np.mean(UpperCamelCase__  ), "line_max": max(UpperCamelCase__  )}





def 	lowercase__		(    snake_case_					:Optional[Any]  ):
	__UpperCAmelCase							=      np.mean([c.isalnum() for c in example['''content''']]  )
	return {"alpha_frac": alpha_frac}





def 	lowercase__		(    snake_case_					:List[str]       ,						snake_case_					:Tuple  ):


	if example["hash"] in uniques:
		uniques.remove(example['''hash''']  )
		return True
	else:
		return False





def 	lowercase__		(    snake_case_					:Tuple       ,						snake_case_					:Dict=5  ):
	__UpperCAmelCase							=      ['''auto-generated''', '''autogenerated''', '''automatically generated''']
	__UpperCAmelCase							=      example['''content'''].splitlines()
	for _, line in zip(range(UpperCamelCase__  )       ,						UpperCamelCase__  ):
		for keyword in keywords:
			if keyword in line.lower():
				return {"autogenerated": True}
	else:
		return {"autogenerated": False}





def 	lowercase__		(    snake_case_					:str       ,						snake_case_					:Any=5       ,						snake_case_					:List[str]=0.05  ):
	__UpperCAmelCase							=      ['''unit tests''', '''test file''', '''configuration file''']
	__UpperCAmelCase							=      example['''content'''].splitlines()
	__UpperCAmelCase							=      0
	__UpperCAmelCase							=      0
	# first test
	for _, line in zip(range(UpperCamelCase__  )       ,						UpperCamelCase__  ):
		for keyword in keywords:
			if keyword in line.lower():
				return {"config_or_test": True}
    # second test
	__UpperCAmelCase							=      example['''content'''].count('''\n'''  )
	__UpperCAmelCase							=      int(coeff * nlines  )
	for line in lines:
		count_config += line.lower().count('''config'''  )
		count_test += line.lower().count('''test'''  )
		if count_config > threshold or count_test > threshold:
			return {"config_or_test": True}
	return {"config_or_test": False}





def 	lowercase__		(    snake_case_					:List[str]  ):
	__UpperCAmelCase							=      ['''def ''', '''class ''', '''for ''', '''while ''']
	__UpperCAmelCase							=      example['''content'''].splitlines()
	for line in lines:
		for keyword in keywords:
			if keyword in line.lower():
				return {"has_no_keywords": False}
	return {"has_no_keywords": True}





def 	lowercase__		(    snake_case_					:Optional[Any]       ,						snake_case_					:List[Any]=4  ):
	__UpperCAmelCase							=      example['''content'''].splitlines()
	__UpperCAmelCase							=      0
	for line in lines:
		counter += line.lower().count('''='''  )
		if counter > minimum:
			return {"has_few_assignments": False}
	return {"has_few_assignments": True}





def 	lowercase__		(    snake_case_					:Optional[int]  ):
	__UpperCAmelCase							=      tokenizer(example['''content''']       ,						truncation=UpperCamelCase__  )['''input_ids''']
	__UpperCAmelCase							=      len(example['''content''']  ) / len(UpperCamelCase__  )
	return {"ratio": ratio}





def 	lowercase__		(    snake_case_					:Dict  ):
	__UpperCAmelCase							=      {}
	results.update(get_hash(UpperCamelCase__  )  )
	results.update(line_stats(UpperCamelCase__  )  )
	results.update(alpha_stats(UpperCamelCase__  )  )
	results.update(char_token_ratio(UpperCamelCase__  )  )
	results.update(is_autogenerated(UpperCamelCase__  )  )
	results.update(is_config_or_test(UpperCamelCase__  )  )
	results.update(has_no_keywords(UpperCamelCase__  )  )
	results.update(has_few_assignments(UpperCamelCase__  )  )
	return results





def 	lowercase__		(    snake_case_					:List[str]       ,						snake_case_					:str       ,						snake_case_					:Optional[Any]  ):


	if not check_uniques(UpperCamelCase__       ,						UpperCamelCase__  ):
		return False
	elif example["autogenerated"]:
		return False
	elif example["line_max"] > args.line_max:
		return False
	elif example["line_mean"] > args.line_mean:
		return False
	elif example["alpha_frac"] < args.alpha_frac:
		return False
	elif example["ratio"] < args.min_token_ratio:
		return False
	elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
		return False
	elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
		return False
	elif example["has_few_assignments"]:
		return False
	else:
		return True





def 	lowercase__		(    snake_case_					:int  ):


	with open(UpperCamelCase__       ,						'''rb'''  ) as f_in:
		with gzip.open(str(UpperCamelCase__  ) + '''.gz'''       ,						'''wb'''       ,						compresslevel=6  ) as f_out:
			shutil.copyfileobj(UpperCamelCase__       ,						UpperCamelCase__  )
	os.unlink(UpperCamelCase__  )


# Settings
_lowercase					:     List[Any]				 =		HfArgumentParser(PreprocessingArguments)
_lowercase					:     Any				 =		parser.parse_args()
if args.num_workers is None:
				_lowercase					:     Tuple				 =		multiprocessing.cpu_count()
_lowercase					:     Optional[int]				 =		AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
_lowercase					:     Tuple				 =		time.time()
_lowercase					:     Union[str, Any]				 =		load_dataset(args.dataset_name, split='train')
print(f"""Time to load dataset: {time.time()-t_start:.2f}""")

# Run preprocessing
_lowercase					:     Optional[Any]				 =		time.time()
_lowercase					:     Dict				 =		ds.map(preprocess, num_proc=args.num_workers)
print(f"""Time to preprocess dataset: {time.time()-t_start:.2f}""")

# Deduplicate hashes
_lowercase					:     int				 =		set(ds.unique('hash'))
_lowercase					:     Optional[Any]				 =		len(uniques) / len(ds)
print(f"""Fraction of duplicates: {1-frac:.2%}""")

# Deduplicate data and apply heuristics
_lowercase					:     int				 =		time.time()
_lowercase					:     Tuple				 =		ds.filter(filter, fn_kwargs={'uniques': uniques, 'args': args})
print(f"""Time to filter dataset: {time.time()-t_start:.2f}""")
print(f"""Size of filtered dataset: {len(ds_filter)}""")

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
				_lowercase					:     Tuple				 =		time.time()
				_lowercase ,_lowercase					:     List[str]				 =		deduplicate_dataset(ds_filter, args.jaccard_threshold)
				print(f"""Time to deduplicate dataset: {time.time()-t_start:.2f}""")
				print(f"""Size of deduplicate dataset: {len(ds_filter)}""")

# Save data in batches of samples_per_file
_lowercase					:     Tuple				 =		Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
				with open(output_dir / 'duplicate_clusters.json', 'w') as f:
								json.dump(duplicate_clusters, f)

_lowercase					:     int				 =		output_dir / 'data'
data_dir.mkdir(exist_ok=True)

_lowercase					:     Optional[int]				 =		time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
				_lowercase					:     Dict				 =		str(data_dir / f"""file-{file_number+1:012}.json""")
				_lowercase					:     Any				 =		min(len(ds_filter), index + args.samples_per_file)
				ds_filter.select(list(range(index, end_index))).to_json(file_path)
				compress_file(file_path)




print(f"""Time to save dataset: {time.time()-t_start:.2f}""")






import argparse

import fairseq
import torch
from torch import nn

from transformers import (
    MBartaaTokenizer,
    MBartConfig,
    MBartForCausalLM,
    SpeechEncoderDecoderConfig,
    SpeechEncoderDecoderModel,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaModel,
    logging,
)


logging.set_verbosity_info()
__A        =    logging.get_logger(__name__)

__A        =    {
    "post_extract_proj": "feature_projection.projection",
    "encoder.pos_conv.0": "encoder.pos_conv_embed.conv",
    "self_attn.k_proj": "encoder.layers.*.attention.k_proj",
    "self_attn.v_proj": "encoder.layers.*.attention.v_proj",
    "self_attn.q_proj": "encoder.layers.*.attention.q_proj",
    "self_attn.out_proj": "encoder.layers.*.attention.out_proj",
    "self_attn_layer_norm": "encoder.layers.*.layer_norm",
    "fc1": "encoder.layers.*.feed_forward.intermediate_dense",
    "fc2": "encoder.layers.*.feed_forward.output_dense",
    "final_layer_norm": "encoder.layers.*.final_layer_norm",
    "encoder.layer_norm": "encoder.layer_norm",
    "w2v_model.layer_norm": "feature_projection.layer_norm",
    "quantizer.weight_proj": "quantizer.weight_proj",
    "quantizer.vars": "quantizer.codevectors",
    "project_q": "project_q",
    "final_proj": "project_hid",
    "w2v_encoder.proj": "lm_head",
    "mask_emb": "masked_spec_embed",
}
__A        =    [
    "lm_head",
    "quantizer.weight_proj",
    "quantizer.codevectors",
    "project_q",
    "project_hid",
]







def 			lowerCamelCase_		(       UpperCamelCase__				:      Any ,     UpperCamelCase__				:      Any ,     UpperCamelCase__				:      Optional[Any] ,     UpperCamelCase__				:      Union[str, Any] ,     UpperCamelCase__				:      Optional[int]      )     ->       Tuple:





				"""simple docstring"""


				for attribute in key.split('.'      ):
								__lowerCamelCase		       =    getattr(UpperCamelCase__ ,     UpperCamelCase__      )

				if weight_type is not None:
								__lowerCamelCase		       =    getattr(UpperCamelCase__ ,     UpperCamelCase__      ).shape
				else:
								__lowerCamelCase		       =    hf_pointer.shape

				assert hf_shape == value.shape, (
				    F"""Shape of hf {key + '.' + weight_type if weight_type is not None else ''} is {hf_shape}, but should be"""
				    F""" {value.shape} for {full_name}"""
				)

				if weight_type == "weight":
								__lowerCamelCase		       =    value
				elif weight_type == "weight_g":
								__lowerCamelCase		       =    value
				elif weight_type == "weight_v":
								__lowerCamelCase		       =    value
				elif weight_type == "bias":
								__lowerCamelCase		       =    value
				else:
								__lowerCamelCase		       =    value

				logger.info(F"""{key + '.' + weight_type if weight_type is not None else ''} was initialized from {full_name}."""      )







def 			lowerCamelCase_		(       UpperCamelCase__				:      Optional[Any] ,     UpperCamelCase__				:      Tuple      )     ->       Optional[Any]:





				"""simple docstring"""


				__lowerCamelCase		       =    []
				__lowerCamelCase		       =    fairseq_model.state_dict()

				__lowerCamelCase		       =    hf_model.feature_extractor
				__lowerCamelCase		       =    hf_model.adapter

				for name, value in fairseq_dict.items():
								__lowerCamelCase		       =    False
								if "conv_layers" in name:
												load_conv_layer(
												    UpperCamelCase__ ,     UpperCamelCase__ ,     UpperCamelCase__ ,     UpperCamelCase__ ,     hf_model.config.feat_extract_norm == 'group' ,     )
												__lowerCamelCase		       =    True
								elif any(x in name for x in ['adaptor', 'w2v_encoder.proj.', 'w2v_proj_ln.']      ):
												load_adapter(UpperCamelCase__ ,     UpperCamelCase__ ,     UpperCamelCase__ ,     UpperCamelCase__      )
												__lowerCamelCase		       =    True
								else:
												for key, mapped_key in MAPPING.items():
																if key in name or key.split('w2v_model.'      )[-1] == name.split('.'      )[0]:
																				__lowerCamelCase		       =    True
																				if "*" in mapped_key:
																								__lowerCamelCase		       =    name.split(UpperCamelCase__      )[0].split('.'      )[-2]
																								__lowerCamelCase		       =    mapped_key.replace('*' ,     UpperCamelCase__      )
																				if "weight_g" in name:
																								__lowerCamelCase		       =    'weight_g'
																				elif "weight_v" in name:
																								__lowerCamelCase		       =    'weight_v'
																				elif "bias" in name:
																								__lowerCamelCase		       =    'bias'
																				elif "weight" in name:
																								__lowerCamelCase		       =    'weight'
																				else:
																								__lowerCamelCase		       =    None
																				set_recursively(UpperCamelCase__ ,     UpperCamelCase__ ,     UpperCamelCase__ ,     UpperCamelCase__ ,     UpperCamelCase__      )
																continue
								if not is_used:
												unused_weights.append(UpperCamelCase__      )

				logger.warning(F"""Unused weights: {unused_weights}"""      )







def 			lowerCamelCase_		(       UpperCamelCase__				:      Dict ,     UpperCamelCase__				:      Any ,     UpperCamelCase__				:      str ,     UpperCamelCase__				:      Tuple ,     UpperCamelCase__				:      Tuple      )     ->       int:





				"""simple docstring"""


				__lowerCamelCase		       =    full_name.split('conv_layers.'      )[-1]
				__lowerCamelCase		       =    name.split('.'      )
				__lowerCamelCase		       =    int(items[0]      )
				__lowerCamelCase		       =    int(items[1]      )

				if type_id == 0:
								if "bias" in name:
												assert value.shape == feature_extractor.conv_layers[layer_id].conv.bias.data.shape, (
												    F"""{full_name} has size {value.shape}, but"""
												    F""" {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found."""
												)
												__lowerCamelCase		       =    value
												logger.info(F"""Feat extract conv layer {layer_id} was initialized from {full_name}."""      )
								elif "weight" in name:
												assert value.shape == feature_extractor.conv_layers[layer_id].conv.weight.data.shape, (
												    F"""{full_name} has size {value.shape}, but"""
												    F""" {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found."""
												)
												__lowerCamelCase		       =    value
												logger.info(F"""Feat extract conv layer {layer_id} was initialized from {full_name}."""      )
				elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
								if "bias" in name:
												assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape, (
												    F"""{full_name} has size {value.shape}, but {feature_extractor[layer_id].layer_norm.bias.data.shape} was"""
												    " found."
												)
												__lowerCamelCase		       =    value
												logger.info(F"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}."""      )
								elif "weight" in name:
												assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape, (
												    F"""{full_name} has size {value.shape}, but"""
												    F""" {feature_extractor[layer_id].layer_norm.weight.data.shape} was found."""
												)
												__lowerCamelCase		       =    value
												logger.info(F"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}."""      )
				else:
								unused_weights.append(UpperCamelCase__      )







def 			lowerCamelCase_		(       UpperCamelCase__				:      Tuple ,     UpperCamelCase__				:      Optional[Any] ,     UpperCamelCase__				:      Any ,     UpperCamelCase__				:      int      )     ->       Union[str, Any]:





				"""simple docstring"""


				__lowerCamelCase		       =    full_name.split('adaptor.'      )[-1]
				__lowerCamelCase		       =    name.split('.'      )

				if items[1].isdigit():
								__lowerCamelCase		       =    int(items[1]      )
				else:
								__lowerCamelCase		       =    None

				if "adaptor" not in full_name:
								if "proj_ln" in full_name:
												# has to be layer norm
												if "bias" in name:
																assert (
																    value.shape == adapter.proj_layer_norm.bias.data.shape
																), F"""{full_name} has size {value.shape}, but {adapter.proj_layer_norm.bias.data.shape} was found."""
																__lowerCamelCase		       =    value
																logger.info(F"""Adapter proj layer norm bias was initialized from {full_name}."""      )
												if "weight" in name:
																assert (
																    value.shape == adapter.proj_layer_norm.weight.data.shape
																), F"""{full_name} has size {value.shape}, but {adapter.proj_layer_norm.weight.data.shape} was found."""
																__lowerCamelCase		       =    value
								else:
												# has to be projection layer
												if "bias" in name:
																assert (
																    value.shape == adapter.proj.bias.data.shape
																), F"""{full_name} has size {value.shape}, but {adapter.proj.bias.data.shape} was found."""
																__lowerCamelCase		       =    value
																logger.info(F"""Adapter proj layer bias was initialized from {full_name}."""      )
												if "weight" in name:
																assert (
																    value.shape == adapter.proj.weight.data.shape
																), F"""{full_name} has size {value.shape}, but {adapter.proj.weight.data.shape} was found."""
																__lowerCamelCase		       =    value
																logger.info(F"""Adapter proj layer weight was initialized from {full_name}."""      )
				elif isinstance(UpperCamelCase__ ,     UpperCamelCase__      ):
								if "bias" in name:
												assert (
												    value.shape == adapter.layers[layer_id].conv.bias.data.shape
												), F"""{full_name} has size {value.shape}, but {adapter.layers[layer_id].conv.bias.data.shape} was found."""
												__lowerCamelCase		       =    value
												logger.info(F"""Adapter layer {layer_id} bias was initialized from {full_name}."""      )
								elif "weight" in name:
												assert (
												    value.shape == adapter.layers[layer_id].conv.weight.data.shape
												), F"""{full_name} has size {value.shape}, but {adapter.layers[layer_id].conv.weight.data.shape} was found."""
												__lowerCamelCase		       =    value
												logger.info(F"""Adapter layer {layer_id} bias was initialized from {full_name}."""      )
				else:
								unused_weights.append(UpperCamelCase__      )







def 			lowerCamelCase_		(       UpperCamelCase__				:      Tuple      )     ->       Tuple:





				"""simple docstring"""


				__lowerCamelCase       ,						__lowerCamelCase		       =    emb.weight.shape
				__lowerCamelCase		       =    nn.Linear(UpperCamelCase__ ,     UpperCamelCase__ ,     bias=UpperCamelCase__      )
				__lowerCamelCase		       =    emb.weight.data
				return lin_layer







@torch.no_grad()
def 			lowerCamelCase_		(       UpperCamelCase__				:      List[str] ,     UpperCamelCase__				:      List[Any] ,     UpperCamelCase__				:      Union[str, Any] ,     UpperCamelCase__				:      Dict ,     UpperCamelCase__				:      List[str] ,     UpperCamelCase__				:      Union[str, Any] ,     UpperCamelCase__				:      Optional[int] ,     UpperCamelCase__				:      Union[str, Any] ,     UpperCamelCase__				:      Optional[Any] ,     UpperCamelCase__				:      str ,     UpperCamelCase__				:      Optional[Any] ,     )     ->       str:





				"""simple docstring"""


				__lowerCamelCase		       =    WavaVecaConfig.from_pretrained(
				    UpperCamelCase__ ,     add_adapter=UpperCamelCase__ ,     adapter_stride=UpperCamelCase__ ,     adapter_kernel_size=UpperCamelCase__ ,     use_auth_token=UpperCamelCase__ ,     output_hidden_size=UpperCamelCase__ ,     )
				__lowerCamelCase		       =    MBartConfig.from_pretrained(UpperCamelCase__      )

				# load model
				__lowerCamelCase       ,						__lowerCamelCase       ,						__lowerCamelCase		       =    fairseq.checkpoint_utils.load_model_ensemble_and_task(
				    [checkpoint_path] ,     arg_overrides={
				        'config_yaml': config_yaml_path,
				        'data': '/'.join(dict_path.split('/'      )[:-1]      ),
				        'w2v_path': checkpoint_path,
				        'load_pretrained_decoder_from': None,
				    } ,     )
				__lowerCamelCase		       =    model[0].eval()

				# load feature extractor
				__lowerCamelCase		       =    WavaVecaFeatureExtractor.from_pretrained(UpperCamelCase__ ,     use_auth_token=UpperCamelCase__      )

				# set weights for wav2vec2 encoder
				__lowerCamelCase		       =    WavaVecaModel(UpperCamelCase__      )

				recursively_load_weights_wavaveca(model.encoder ,     UpperCamelCase__      )

				# load decoder weights
				__lowerCamelCase		       =    MBartForCausalLM(UpperCamelCase__      )
				__lowerCamelCase       ,						__lowerCamelCase		       =    hf_decoder.model.decoder.load_state_dict(model.decoder.state_dict() ,     strict=UpperCamelCase__      )
				logger.warning(F"""The following keys are missing when loading the decoder weights: {missing_keys}"""      )
				logger.warning(F"""The following keys are unexpected when loading the decoder weights: {unexpected_keys}"""      )

				__lowerCamelCase		       =    SpeechEncoderDecoderModel(encoder=UpperCamelCase__ ,     decoder=UpperCamelCase__      )
				__lowerCamelCase		       =    False

				__lowerCamelCase		       =    MBartaaTokenizer(UpperCamelCase__      )
				tokenizer.save_pretrained(UpperCamelCase__      )

				__lowerCamelCase		       =    hf_wavavec.config.to_dict()
				__lowerCamelCase		       =    tokenizer.pad_token_id
				__lowerCamelCase		       =    tokenizer.bos_token_id
				__lowerCamelCase		       =    tokenizer.eos_token_id
				__lowerCamelCase		       =    'mbart50'
				__lowerCamelCase		       =    'wav2vec2'

				__lowerCamelCase		       =    tokenizer.eos_token_id
				__lowerCamelCase		       =    25_0004
				__lowerCamelCase		       =    tokenizer.eos_token_id

				__lowerCamelCase		       =    SpeechEncoderDecoderConfig.from_dict(UpperCamelCase__      )

				hf_wavavec.save_pretrained(UpperCamelCase__      )
				feature_extractor.save_pretrained(UpperCamelCase__      )


if __name__ == "__main__":
					__A        =    argparse.ArgumentParser()
					parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.")
					parser.add_argument("--checkpoint_path", default=None, type=str, help="Path to fairseq checkpoint")
					parser.add_argument("--dict_path", default=None, type=str, help="Path to dict of fine-tuned model")
					parser.add_argument("--config_yaml_path", default=None, type=str, help="Path to yaml file of fine-tuned model")
					parser.add_argument(
					    "--encoder_config_path",
					    default="facebook/wav2vec2-xls-r-1b",
					    type=str,
					    help="Path to hf encoder wav2vec2 checkpoint config",
					)
					parser.add_argument(
					    "--decoder_config_path",
					    default="facebook/mbart-large-50-one-to-many-mmt",
					    type=str,
					    help="Path to hf decoder checkpoint config",
					)
					parser.add_argument("--add_adapter", default=True, type=bool, help="whethere to add model adapter layers")
					parser.add_argument("--adapter_stride", default=2, type=int, help="stride of adapter layers")
					parser.add_argument("--adapter_kernel_size", default=3, type=int, help="kernel size of adapter layers")
					parser.add_argument("--encoder_output_dim", default=10_24, type=int, help="encoder output dim")
					parser.add_argument("--start_token_id", default=25_00_04, type=int, help="`decoder_start_token_id` of model config")

					__A        =    parser.parse_args()
					convert_wavaveca_checkpoint(
					    args.checkpoint_path,
					    args.pytorch_dump_folder_path,
					    args.dict_path,
					    args.config_yaml_path,
					    encoder_config_path=args.encoder_config_path,
					    decoder_config_path=args.decoder_config_path,
					    add_adapter=args.add_adapter,
					    adapter_kernel_size=args.adapter_kernel_size,
					    adapter_stride=args.adapter_stride,
					    decoder_start_token_id=args.start_token_id,
					    encoder_output_dim=args.encoder_output_dim,
					)


