





import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


lowerCAmelCase_     =     re.compile(r"""\s+""")


def      lowerCamelCase_						(   lowerCAmelCase:						Union[str, Any]					)->		Optional[Any]:
  return {"hash": hashlib.mda(re.sub(SCREAMING_SNAKE_CASE_     ,				''     ,				example['content']					).encode('utf-8'					)					).hexdigest()}


def      lowerCamelCase_						(   lowerCAmelCase:						List[str]					)->		Union[str, Any]:
  _snake_case :					str								=						[len(SCREAMING_SNAKE_CASE_					) for line in example['content'].splitlines()]
  return {"line_mean": np.mean(SCREAMING_SNAKE_CASE_					), "line_max": max(SCREAMING_SNAKE_CASE_					)}


def      lowerCamelCase_						(   lowerCAmelCase:						Tuple					)->		List[Any]:
  _snake_case :					int								=						np.mean([c.isalnum() for c in example['content']]					)
  return {"alpha_frac": alpha_frac}


def      lowerCamelCase_						(   lowerCAmelCase:						Dict     ,				lowerCAmelCase:						Optional[Any]					)->		Tuple:






  if example["hash"] in uniques:
    uniques.remove(example['hash']					)
    return True
  else:
    return False


def      lowerCamelCase_						(   lowerCAmelCase:						int     ,				lowerCAmelCase:						str=5					)->		Union[str, Any]:
  _snake_case :					Any								=						['auto-generated', 'autogenerated', 'automatically generated']
  _snake_case :					Optional[Any]								=						example['content'].splitlines()
  for _, line in zip(range(SCREAMING_SNAKE_CASE_					)     ,				SCREAMING_SNAKE_CASE_					):
    for keyword in keywords:
      if keyword in line.lower():
        return {"autogenerated": True}
  else:
    return {"autogenerated": False}


def      lowerCamelCase_						(   lowerCAmelCase:						List[str]     ,				lowerCAmelCase:						List[Any]=5     ,				lowerCAmelCase:						Dict=0.0_5					)->		Any:
  _snake_case :					Optional[Any]								=						['unit tests', 'test file', 'configuration file']
  _snake_case :					Union[str, Any]								=						example['content'].splitlines()
  _snake_case :					int								=						0
  _snake_case :					List[str]								=						0
  # first test
  for _, line in zip(range(SCREAMING_SNAKE_CASE_					)     ,				SCREAMING_SNAKE_CASE_					):
    for keyword in keywords:
      if keyword in line.lower():
        return {"config_or_test": True}
    # second test
  _snake_case :					str								=						example['content'].count('\n'					)
  _snake_case :					List[str]								=						int(coeff * nlines					)
  for line in lines:
    count_config += line.lower().count('config'					)
    count_test += line.lower().count('test'					)
    if count_config > threshold or count_test > threshold:
      return {"config_or_test": True}
  return {"config_or_test": False}


def      lowerCamelCase_						(   lowerCAmelCase:						List[str]					)->		Union[str, Any]:
  _snake_case :					Union[str, Any]								=						['def ', 'class ', 'for ', 'while ']
  _snake_case :					List[Any]								=						example['content'].splitlines()
  for line in lines:
    for keyword in keywords:
      if keyword in line.lower():
        return {"has_no_keywords": False}
  return {"has_no_keywords": True}


def      lowerCamelCase_						(   lowerCAmelCase:						int     ,				lowerCAmelCase:						List[str]=4					)->		List[Any]:
  _snake_case :					int								=						example['content'].splitlines()
  _snake_case :					List[str]								=						0
  for line in lines:
    counter += line.lower().count('='					)
    if counter > minimum:
      return {"has_few_assignments": False}
  return {"has_few_assignments": True}


def      lowerCamelCase_						(   lowerCAmelCase:						List[Any]					)->		Tuple:
  _snake_case :					Union[str, Any]								=						tokenizer(example['content']     ,				truncation=SCREAMING_SNAKE_CASE_					)['input_ids']
  _snake_case :					List[Any]								=						len(example['content']					) / len(SCREAMING_SNAKE_CASE_					)
  return {"ratio": ratio}


def      lowerCamelCase_						(   lowerCAmelCase:						str					)->		List[Any]:
  _snake_case :					Tuple								=						{}
  results.update(get_hash(SCREAMING_SNAKE_CASE_					)					)
  results.update(line_stats(SCREAMING_SNAKE_CASE_					)					)
  results.update(alpha_stats(SCREAMING_SNAKE_CASE_					)					)
  results.update(char_token_ratio(SCREAMING_SNAKE_CASE_					)					)
  results.update(is_autogenerated(SCREAMING_SNAKE_CASE_					)					)
  results.update(is_config_or_test(SCREAMING_SNAKE_CASE_					)					)
  results.update(has_no_keywords(SCREAMING_SNAKE_CASE_					)					)
  results.update(has_few_assignments(SCREAMING_SNAKE_CASE_					)					)
  return results


def      lowerCamelCase_						(   lowerCAmelCase:						int     ,				lowerCAmelCase:						Optional[Any]     ,				lowerCAmelCase:						str					)->		Tuple:






  if not check_uniques(SCREAMING_SNAKE_CASE_     ,				SCREAMING_SNAKE_CASE_					):
    return False
  elif example["autogenerated"]:
    return False
  elif example["line_max"] > args.line_max:
    return False
  elif example["line_mean"] > args.line_mean:
    return False
  elif example["alpha_frac"] < args.alpha_frac:
    return False
  elif example["ratio"] < args.min_token_ratio:
    return False
  elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
    return False
  elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
    return False
  elif example["has_few_assignments"]:
    return False
  else:
    return True


def      lowerCamelCase_						(   lowerCAmelCase:						List[Any]					)->		List[Any]:






  with open(SCREAMING_SNAKE_CASE_     ,				'rb'					) as f_in:
    with gzip.open(str(SCREAMING_SNAKE_CASE_					) + '.gz'     ,				'wb'     ,				compresslevel=6					) as f_out:
      shutil.copyfileobj(SCREAMING_SNAKE_CASE_     ,				SCREAMING_SNAKE_CASE_					)
  os.unlink(SCREAMING_SNAKE_CASE_					)


# Settings
lowerCAmelCase_     =     HfArgumentParser(PreprocessingArguments)
lowerCAmelCase_     =     parser.parse_args()
if args.num_workers is None:
      lowerCAmelCase_     =     multiprocessing.cpu_count()
lowerCAmelCase_     =     AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
lowerCAmelCase_     =     time.time()
lowerCAmelCase_     =     load_dataset(args.dataset_name, split="""train""")
print(F"""Time to load dataset: {time.time()-t_start:.2f}""")

# Run preprocessing
lowerCAmelCase_     =     time.time()
lowerCAmelCase_     =     ds.map(preprocess, num_proc=args.num_workers)
print(F"""Time to preprocess dataset: {time.time()-t_start:.2f}""")

# Deduplicate hashes
lowerCAmelCase_     =     set(ds.unique("""hash"""))
lowerCAmelCase_     =     len(uniques) / len(ds)
print(F"""Fraction of duplicates: {1-frac:.2%}""")

# Deduplicate data and apply heuristics
lowerCAmelCase_     =     time.time()
lowerCAmelCase_     =     ds.filter(filter, fn_kwargs={"""uniques""": uniques, """args""": args})
print(F"""Time to filter dataset: {time.time()-t_start:.2f}""")
print(F"""Size of filtered dataset: {len(ds_filter)}""")

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
      lowerCAmelCase_     =     time.time()
      lowerCAmelCase_     =     deduplicate_dataset(ds_filter, args.jaccard_threshold)
      print(F"""Time to deduplicate dataset: {time.time()-t_start:.2f}""")
      print(F"""Size of deduplicate dataset: {len(ds_filter)}""")

# Save data in batches of samples_per_file
lowerCAmelCase_     =     Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
      with open(output_dir / """duplicate_clusters.json""", """w""") as f:
            json.dump(duplicate_clusters, f)

lowerCAmelCase_     =     output_dir / """data"""
data_dir.mkdir(exist_ok=True)

lowerCAmelCase_     =     time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
      lowerCAmelCase_     =     str(data_dir / F"""file-{file_number+1:012}.json""")
      lowerCAmelCase_     =     min(len(ds_filter), index + args.samples_per_file)
      ds_filter.select(list(range(index, end_index))).to_json(file_path)
      compress_file(file_path)







print(F"""Time to save dataset: {time.time()-t_start:.2f}""")







import math
from collections import defaultdict
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput




def 			lowerCAmelCase(      SCREAMING_SNAKE_CASE_			,						SCREAMING_SNAKE_CASE_=0.999			,						SCREAMING_SNAKE_CASE_="cosine"			,						)->						int:



    """simple docstring"""






    if alpha_transform_type == "cosine":

        def alpha_bar_fn(SCREAMING_SNAKE_CASE_			):
            return math.cos((t + 0.008) / 1.008 * math.pi / 2			) ** 2

    elif alpha_transform_type == "exp":

        def alpha_bar_fn(SCREAMING_SNAKE_CASE_			):
            return math.exp(t * -12.0			)

    else:
        raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}"			)

    UpperCamelCase_			  =						[]
    for i in range(SCREAMING_SNAKE_CASE_			):
        UpperCamelCase_			  =						i / num_diffusion_timesteps
        UpperCamelCase_			  =						(i + 1) / num_diffusion_timesteps
        betas.append(min(1 - alpha_bar_fn(SCREAMING_SNAKE_CASE_			) / alpha_bar_fn(SCREAMING_SNAKE_CASE_			)			,						SCREAMING_SNAKE_CASE_			)			)
    return torch.tensor(SCREAMING_SNAKE_CASE_			,						dtype=torch.floataa			)






class 						__magic_name__					(							snake_case			,      snake_case				):
     UpperCamelCase_	:str										=      [e.name for e in KarrasDiffusionSchedulers]
     UpperCamelCase_	:Tuple										=      2






     @register_to_config
     def __init__(				self		,						_lowercase = 1_000		,						_lowercase = 0.00_085		,						_lowercase = 0.012		,						_lowercase = "linear"		,						_lowercase = None		,						_lowercase = "epsilon"		,						_lowercase = "linspace"		,						_lowercase = 0		,						)->     List[Any]:
         if trained_betas is not None:
             UpperCamelCase_			  =						torch.tensor(_lowercase		,						dtype=torch.floataa	)
         elif beta_schedule == "linear":
             UpperCamelCase_			  =						torch.linspace(_lowercase		,						_lowercase		,						_lowercase		,						dtype=torch.floataa	)
         elif beta_schedule == "scaled_linear":
             # this schedule is very specific to the latent diffusion model.
             UpperCamelCase_			  =						(
                 torch.linspace(beta_start**0.5		,						beta_end**0.5		,						_lowercase		,						dtype=torch.floataa	) ** 2
             )
         elif beta_schedule == "squaredcos_cap_v2":
             # Glide cosine schedule
             UpperCamelCase_			  =						betas_for_alpha_bar(_lowercase	)
         else:
             raise NotImplementedError(F"{beta_schedule} does is not implemented for {self.__class__}"	)

         UpperCamelCase_			  =						1.0 - self.betas
         UpperCamelCase_			  =						torch.cumprod(self.alphas		,						dim=0	)

         #  set all values
         self.set_timesteps(_lowercase		,						_lowercase		,						_lowercase	)






     def 	UpperCAmelCase_				(				self		,						_lowercase		,						_lowercase=None	)->     Union[str, Any]:
         if schedule_timesteps is None:
             UpperCamelCase_			  =						self.timesteps

         UpperCamelCase_			  =						(schedule_timesteps == timestep).nonzero()

         # The sigma index that is taken for the **very** first `step`
         # is always the second index (or the last index if there is only 1)
         # This way we can ensure we don't accidentally skip a sigma in
         # case we start in the middle of the denoising schedule (e.g. for image-to-image)
         if len(self._index_counter	) == 0:
             UpperCamelCase_			  =						1 if len(_lowercase	) > 1 else 0
         else:
             UpperCamelCase_			  =						timestep.cpu().item() if torch.is_tensor(_lowercase	) else timestep
             UpperCamelCase_			  =						self._index_counter[timestep_int]

         return indices[pos].item()






     @property
     def 	UpperCAmelCase_				(				self	)->     Tuple:
         # standard deviation of the initial noise distribution
         if self.config.timestep_spacing in ["linspace", "trailing"]:
             return self.sigmas.max()

         return (self.sigmas.max() ** 2 + 1) ** 0.5






     def 	UpperCAmelCase_				(				self		,						_lowercase		,						_lowercase		,						)->     torch.FloatTensor:
         UpperCamelCase_			  =						self.index_for_timestep(_lowercase	)

         if self.state_in_first_order:
             UpperCamelCase_			  =						self.sigmas[step_index]
         else:
             UpperCamelCase_			  =						self.sigmas_interpol[step_index]

         UpperCamelCase_			  =						sample / ((sigma**2 + 1) ** 0.5)
         return sample






     def 	UpperCAmelCase_				(				self		,						_lowercase		,						_lowercase = None		,						_lowercase = None		,						)->     Tuple:
         UpperCamelCase_			  =						num_inference_steps

         UpperCamelCase_			  =						num_train_timesteps or self.config.num_train_timesteps

         # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891
         if self.config.timestep_spacing == "linspace":
             UpperCamelCase_			  =						np.linspace(0		,						num_train_timesteps - 1		,						_lowercase		,						dtype=_lowercase	)[::-1].copy()
         elif self.config.timestep_spacing == "leading":
             UpperCamelCase_			  =						num_train_timesteps // self.num_inference_steps
             # creates integer timesteps by multiplying by ratio
             # casting to int to avoid issues when num_inference_step is power of 3
             UpperCamelCase_			  =						(np.arange(0		,						_lowercase	) * step_ratio).round()[::-1].copy().astype(_lowercase	)
             timesteps += self.config.steps_offset
         elif self.config.timestep_spacing == "trailing":
             UpperCamelCase_			  =						num_train_timesteps / self.num_inference_steps
             # creates integer timesteps by multiplying by ratio
             # casting to int to avoid issues when num_inference_step is power of 3
             UpperCamelCase_			  =						(np.arange(_lowercase		,						0		,						-step_ratio	)).round().copy().astype(_lowercase	)
             timesteps -= 1
         else:
             raise ValueError(
                 F"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'."	)

         UpperCamelCase_			  =						np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5	)
         UpperCamelCase_			  =						torch.from_numpy(np.log(_lowercase	)	).to(_lowercase	)

         UpperCamelCase_			  =						np.interp(_lowercase		,						np.arange(0		,						len(_lowercase	)	)		,						_lowercase	)
         UpperCamelCase_			  =						np.concatenate([sigmas, [0.0]]	).astype(np.floataa	)
         UpperCamelCase_			  =						torch.from_numpy(_lowercase	).to(device=_lowercase	)

         # interpolate sigmas
         UpperCamelCase_			  =						sigmas.log().lerp(sigmas.roll(1	).log()		,						0.5	).exp()

         UpperCamelCase_			  =						torch.cat([sigmas[:1], sigmas[1:].repeat_interleave(2	), sigmas[-1:]]	)
         UpperCamelCase_			  =						torch.cat(
             [sigmas_interpol[:1], sigmas_interpol[1:].repeat_interleave(2	), sigmas_interpol[-1:]]	)

         if str(_lowercase	).startswith("mps"	):
             # mps does not support float64
             UpperCamelCase_			  =						torch.from_numpy(_lowercase	).to(_lowercase		,						dtype=torch.floataa	)
         else:
             UpperCamelCase_			  =						torch.from_numpy(_lowercase	).to(_lowercase	)

         # interpolate timesteps
         UpperCamelCase_			  =						self.sigma_to_t(_lowercase	).to(_lowercase		,						dtype=timesteps.dtype	)
         UpperCamelCase_			  =						torch.stack((timesteps_interpol[1:-1, None], timesteps[1:, None])		,						dim=-1	).flatten()

         UpperCamelCase_			  =						torch.cat([timesteps[:1], interleaved_timesteps]	)

         UpperCamelCase_			  =						None

         # for exp beta schedules, such as the one for `pipeline_shap_e.py`
         # we need an index counter
         UpperCamelCase_			  =						defaultdict(_lowercase	)






     def 	UpperCAmelCase_				(				self		,						_lowercase	)->     Any:
         # get log sigma
         UpperCamelCase_			  =						sigma.log()

         # get distribution
         UpperCamelCase_			  =						log_sigma - self.log_sigmas[:, None]

         # get sigmas range
         UpperCamelCase_			  =						dists.ge(0	).cumsum(dim=0	).argmax(dim=0	).clamp(max=self.log_sigmas.shape[0] - 2	)
         UpperCamelCase_			  =						low_idx + 1

         UpperCamelCase_			  =						self.log_sigmas[low_idx]
         UpperCamelCase_			  =						self.log_sigmas[high_idx]

         # interpolate sigmas
         UpperCamelCase_			  =						(low - log_sigma) / (low - high)
         UpperCamelCase_			  =						w.clamp(0		,						1	)

         # transform interpolation to time range
         UpperCamelCase_			  =						(1 - w) * low_idx + w * high_idx
         UpperCamelCase_			  =						t.view(sigma.shape	)
         return t






     @property
     def 	UpperCAmelCase_				(				self	)->     Any:
         return self.sample is None






     def 	UpperCAmelCase_				(				self		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase = True		,						)->     Union[SchedulerOutput, Tuple]:
         UpperCamelCase_			  =						self.index_for_timestep(_lowercase	)

         # advance index counter by 1
         UpperCamelCase_			  =						timestep.cpu().item() if torch.is_tensor(_lowercase	) else timestep
         self._index_counter[timestep_int] += 1

         if self.state_in_first_order:
             UpperCamelCase_			  =						self.sigmas[step_index]
             UpperCamelCase_			  =						self.sigmas_interpol[step_index + 1]
             UpperCamelCase_			  =						self.sigmas[step_index + 1]
         else:
             # 2nd order / KDPM2's method
             UpperCamelCase_			  =						self.sigmas[step_index - 1]
             UpperCamelCase_			  =						self.sigmas_interpol[step_index]
             UpperCamelCase_			  =						self.sigmas[step_index]

         # currently only gamma=0 is supported. This usually works best anyways.
         # We can support gamma in the future but then need to scale the timestep before
         # passing it to the model which requires a change in API
         UpperCamelCase_			  =						0
         UpperCamelCase_			  =						sigma * (gamma + 1)  # Note: sigma_hat == sigma for now

         # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise
         if self.config.prediction_type == "epsilon":
             UpperCamelCase_			  =						sigma_hat if self.state_in_first_order else sigma_interpol
             UpperCamelCase_			  =						sample - sigma_input * model_output
         elif self.config.prediction_type == "v_prediction":
             UpperCamelCase_			  =						sigma_hat if self.state_in_first_order else sigma_interpol
             UpperCamelCase_			  =						model_output * (-sigma_input / (sigma_input**2 + 1) ** 0.5) + (
                 sample / (sigma_input**2 + 1)
             )
         elif self.config.prediction_type == "sample":
             raise NotImplementedError("prediction_type not implemented yet: sample"	)
         else:
             raise ValueError(
                 F"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`"	)

         if self.state_in_first_order:
             # 2. Convert to an ODE derivative for 1st order
             UpperCamelCase_			  =						(sample - pred_original_sample) / sigma_hat
             # 3. delta timestep
             UpperCamelCase_			  =						sigma_interpol - sigma_hat

             # store for 2nd order step
             UpperCamelCase_			  =						sample
         else:
             # DPM-Solver-2
             # 2. Convert to an ODE derivative for 2nd order
             UpperCamelCase_			  =						(sample - pred_original_sample) / sigma_interpol

             # 3. delta timestep
             UpperCamelCase_			  =						sigma_next - sigma_hat

             UpperCamelCase_			  =						self.sample
             UpperCamelCase_			  =						None

         UpperCamelCase_			  =						sample + derivative * dt

         if not return_dict:
             return (prev_sample,)

         return SchedulerOutput(prev_sample=_lowercase	)






     def 	UpperCAmelCase_				(				self		,						_lowercase		,						_lowercase		,						_lowercase		,						)->     torch.FloatTensor:
         # Make sure sigmas and timesteps have the same device and dtype as original_samples
         UpperCamelCase_			  =						self.sigmas.to(device=original_samples.device		,						dtype=original_samples.dtype	)
         if original_samples.device.type == "mps" and torch.is_floating_point(_lowercase	):
             # mps does not support float64
             UpperCamelCase_			  =						self.timesteps.to(original_samples.device		,						dtype=torch.floataa	)
             UpperCamelCase_			  =						timesteps.to(original_samples.device		,						dtype=torch.floataa	)
         else:
             UpperCamelCase_			  =						self.timesteps.to(original_samples.device	)
             UpperCamelCase_			  =						timesteps.to(original_samples.device	)

         UpperCamelCase_			  =						[self.index_for_timestep(_lowercase		,						_lowercase	) for t in timesteps]

         UpperCamelCase_			  =						sigmas[step_indices].flatten()
         while len(sigma.shape	) < len(original_samples.shape	):
             UpperCamelCase_			  =						sigma.unsqueeze(-1	)

         UpperCamelCase_			  =						original_samples + noise * sigma
         return noisy_samples






     def __len__(				self	)->     Dict:
         return self.config.num_train_timesteps
