
'''simple docstring'''



import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


_lowercase          =						re.compile(R"""\s+""")


def A (__lowerCamelCase :Optional[int]						):
						return {"hash": hashlib.mda(re.sub(__lowerCamelCase					,      """"""					,      example["""content"""]						).encode("""utf-8"""						)						).hexdigest()}


def A (__lowerCamelCase :Optional[Any]						):
						_lowerCAmelCase    		=					[len(__lowerCamelCase						) for line in example["""content"""].splitlines()]
						return {"line_mean": np.mean(__lowerCamelCase						), "line_max": max(__lowerCamelCase						)}


def A (__lowerCamelCase :Tuple						):
						_lowerCAmelCase    		=					np.mean([c.isalnum() for c in example["""content"""]]						)
						return {"alpha_frac": alpha_frac}


def A (__lowerCamelCase :Dict					,      __lowerCamelCase :str						):
						if example["hash"] in uniques:
												uniques.remove(example["""hash"""]						)
												return True
						else:
												return False


def A (__lowerCamelCase :Tuple					,      __lowerCamelCase :Optional[Any]=5						):
						_lowerCAmelCase    		=					["""auto-generated""", """autogenerated""", """automatically generated"""]
						_lowerCAmelCase    		=					example["""content"""].splitlines()
						for _, line in zip(range(__lowerCamelCase						)					,      __lowerCamelCase						):
												for keyword in keywords:
																		if keyword in line.lower():
																								return {"autogenerated": True}
						else:
												return {"autogenerated": False}


def A (__lowerCamelCase :Tuple					,      __lowerCamelCase :Union[str, Any]=5					,      __lowerCamelCase :Dict=0.05						):
						_lowerCAmelCase    		=					["""unit tests""", """test file""", """configuration file"""]
						_lowerCAmelCase    		=					example["""content"""].splitlines()
						_lowerCAmelCase    		=					0
						_lowerCAmelCase    		=					0
						# first test
						for _, line in zip(range(__lowerCamelCase						)					,      __lowerCamelCase						):
												for keyword in keywords:
																		if keyword in line.lower():
																								return {"config_or_test": True}
    # second test
						_lowerCAmelCase    		=					example["""content"""].count("""\n"""						)
						_lowerCAmelCase    		=					int(coeff * nlines						)
						for line in lines:
												count_config += line.lower().count("""config"""						)
												count_test += line.lower().count("""test"""						)
												if count_config > threshold or count_test > threshold:
																		return {"config_or_test": True}
						return {"config_or_test": False}


def A (__lowerCamelCase :Tuple						):
						_lowerCAmelCase    		=					["""def """, """class """, """for """, """while """]
						_lowerCAmelCase    		=					example["""content"""].splitlines()
						for line in lines:
												for keyword in keywords:
																		if keyword in line.lower():
																								return {"has_no_keywords": False}
						return {"has_no_keywords": True}


def A (__lowerCamelCase :Union[str, Any]					,      __lowerCamelCase :int=4						):
						_lowerCAmelCase    		=					example["""content"""].splitlines()
						_lowerCAmelCase    		=					0
						for line in lines:
												counter += line.lower().count("""="""						)
												if counter > minimum:
																		return {"has_few_assignments": False}
						return {"has_few_assignments": True}


def A (__lowerCamelCase :Tuple						):
						_lowerCAmelCase    		=					tokenizer(example["""content"""]					,      truncation=__lowerCamelCase						)["""input_ids"""]
						_lowerCAmelCase    		=					len(example["""content"""]						) / len(__lowerCamelCase						)
						return {"ratio": ratio}


def A (__lowerCamelCase :Any						):
						_lowerCAmelCase    		=					{}
						results.update(get_hash(__lowerCamelCase						)						)
						results.update(line_stats(__lowerCamelCase						)						)
						results.update(alpha_stats(__lowerCamelCase						)						)
						results.update(char_token_ratio(__lowerCamelCase						)						)
						results.update(is_autogenerated(__lowerCamelCase						)						)
						results.update(is_config_or_test(__lowerCamelCase						)						)
						results.update(has_no_keywords(__lowerCamelCase						)						)
						results.update(has_few_assignments(__lowerCamelCase						)						)
						return results


def A (__lowerCamelCase :List[Any]					,      __lowerCamelCase :int					,      __lowerCamelCase :List[Any]						):
						if not check_uniques(__lowerCamelCase					,      __lowerCamelCase						):
												return False
						elif example["autogenerated"]:
												return False
						elif example["line_max"] > args.line_max:
												return False
						elif example["line_mean"] > args.line_mean:
												return False
						elif example["alpha_frac"] < args.alpha_frac:
												return False
						elif example["ratio"] < args.min_token_ratio:
												return False
						elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
												return False
						elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
												return False
						elif example["has_few_assignments"]:
												return False
						else:
												return True


def A (__lowerCamelCase :int						):
						with open(__lowerCamelCase					,      """rb"""						) as f_in:
												with gzip.open(str(__lowerCamelCase						) + """.gz"""					,      """wb"""					,      compresslevel=6						) as f_out:
																		shutil.copyfileobj(__lowerCamelCase					,      __lowerCamelCase						)
						os.unlink(__lowerCamelCase						)


# Settings
_lowercase          =						HfArgumentParser(PreprocessingArguments)
_lowercase          =						parser.parse_args()
if args.num_workers is None:
						_lowercase          =						multiprocessing.cpu_count()
_lowercase          =						AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
_lowercase          =						time.time()
_lowercase          =						load_dataset(args.dataset_name, split="""train""")
print(F"""Time to load dataset: {time.time()-t_start:.2f}""")

# Run preprocessing
_lowercase          =						time.time()
_lowercase          =						ds.map(preprocess, num_proc=args.num_workers)
print(F"""Time to preprocess dataset: {time.time()-t_start:.2f}""")

# Deduplicate hashes
_lowercase          =						set(ds.unique("""hash"""))
_lowercase          =						len(uniques) / len(ds)
print(F"""Fraction of duplicates: {1-frac:.2%}""")

# Deduplicate data and apply heuristics
_lowercase          =						time.time()
_lowercase          =						ds.filter(filter, fn_kwargs={"""uniques""": uniques, """args""": args})
print(F"""Time to filter dataset: {time.time()-t_start:.2f}""")
print(F"""Size of filtered dataset: {len(ds_filter)}""")

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
						_lowercase          =						time.time()
						_lowercase       ,		_lowercase          =						deduplicate_dataset(ds_filter, args.jaccard_threshold)
						print(F"""Time to deduplicate dataset: {time.time()-t_start:.2f}""")
						print(F"""Size of deduplicate dataset: {len(ds_filter)}""")

# Save data in batches of samples_per_file
_lowercase          =						Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
						with open(output_dir / """duplicate_clusters.json""", """w""") as f:
												json.dump(duplicate_clusters, f)

_lowercase          =						output_dir / """data"""
data_dir.mkdir(exist_ok=True)

_lowercase          =						time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
						_lowercase          =						str(data_dir / F"""file-{file_number+1:012}.json""")
						_lowercase          =						min(len(ds_filter), index + args.samples_per_file)
						ds_filter.select(list(range(index, end_index))).to_json(file_path)
						compress_file(file_path)





print(F"""Time to save dataset: {time.time()-t_start:.2f}""")

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE		:     Union[str, Any]         =	logging.get_logger(__name__)


SCREAMING_SNAKE_CASE		:     Union[str, Any]         =	"▁"

SCREAMING_SNAKE_CASE		:     Dict         =	{"vocab_file": "spiece.model"}

SCREAMING_SNAKE_CASE		:     List[Any]         =	{
    "vocab_file": {
        "google/reformer-crime-and-punishment": (
            "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model"
        )
    }
}

SCREAMING_SNAKE_CASE		:     Any         =	{
    "google/reformer-crime-and-punishment": 524288,
}

class        UpperCamelCase						(				__a				):
						a__		:Optional[Any]      =			VOCAB_FILES_NAMES
						a__		:List[Any]      =			PRETRAINED_VOCAB_FILES_MAP
						a__		:Union[str, Any]      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						a__		:List[Any]      =			['''input_ids''', '''attention_mask''']





						def __init__(self			,     __UpperCamelCase			,     __UpperCamelCase="</s>"			,     __UpperCamelCase="<unk>"			,     __UpperCamelCase=[]			,     __UpperCamelCase = None			,     **__UpperCamelCase			,     )				->				None:
												UpperCamelCase_		:							List[str]  =		{} if sp_model_kwargs is None else sp_model_kwargs

												super().__init__(
												    eos_token=__UpperCamelCase			,     unk_token=__UpperCamelCase			,     additional_special_tokens=__UpperCamelCase			,     sp_model_kwargs=self.sp_model_kwargs			,     **__UpperCamelCase			,     )

												UpperCamelCase_		:							int  =		vocab_file
												UpperCamelCase_		:							str  =		spm.SentencePieceProcessor(**self.sp_model_kwargs						)
												self.sp_model.Load(__UpperCamelCase						)





						@property
						def        A_							(self						)				->				Dict:
												return self.sp_model.get_piece_size()





						def        A_							(self						)				->				Dict[str, int]:
												UpperCamelCase_		:							str  =		{self.convert_ids_to_tokens(__UpperCamelCase						): i for i in range(self.vocab_size						)}
												vocab.update(self.added_tokens_encoder						)
												return vocab





						def __getstate__(self						)				->				Any:
												UpperCamelCase_		:							Dict  =		self.__dict__.copy()
												UpperCamelCase_		:							List[str]  =		None
												return state





						def __setstate__(self			,     __UpperCamelCase						)				->				List[Any]:
												UpperCamelCase_		:							List[Any]  =		d

												# for backward compatibility
												if not hasattr(self			,     """sp_model_kwargs"""						):
																		UpperCamelCase_		:							Any  =		{}

												UpperCamelCase_		:							Dict  =		spm.SentencePieceProcessor(**self.sp_model_kwargs						)
												self.sp_model.Load(self.vocab_file						)





						def        A_							(self			,     __UpperCamelCase						)				->				List[str]:
												return self.sp_model.encode(__UpperCamelCase			,     out_type=__UpperCamelCase						)





						def        A_							(self			,     __UpperCamelCase						)				->				Union[str, Any]:
												return self.sp_model.piece_to_id(__UpperCamelCase						)





						def        A_							(self			,     __UpperCamelCase						)				->				Dict:
												if index < self.sp_model.get_piece_size():
																		UpperCamelCase_		:							Union[str, Any]  =		self.sp_model.IdToPiece(__UpperCamelCase						)
												return token





						def        A_							(self			,     __UpperCamelCase						)				->				int:
												UpperCamelCase_		:							Dict  =		[]
												UpperCamelCase_		:							Union[str, Any]  =		""""""
												for token in tokens:
																		# make sure that special tokens are not decoded using sentencepiece model
																		if token in self.all_special_tokens:
																								out_string += self.sp_model.decode(__UpperCamelCase						) + token
																								UpperCamelCase_		:							Optional[Any]  =		[]
																		else:
																								current_sub_tokens.append(__UpperCamelCase						)
												out_string += self.sp_model.decode(__UpperCamelCase						)
												return out_string.strip()





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase = None						)				->				Tuple[str]:
												if not os.path.isdir(__UpperCamelCase						):
																		logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
																		return
												UpperCamelCase_		:							Tuple  =		os.path.join(
												    __UpperCamelCase			,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

												if os.path.abspath(self.vocab_file						) != os.path.abspath(__UpperCamelCase						) and os.path.isfile(self.vocab_file						):
																		copyfile(self.vocab_file			,     __UpperCamelCase						)
												elif not os.path.isfile(self.vocab_file						):
																		with open(__UpperCamelCase			,     """wb"""						) as fi:
																								UpperCamelCase_		:							int  =		self.sp_model.serialized_model_proto()
																								fi.write(__UpperCamelCase						)

												return (out_vocab_file,)






