# What does this PR do?
Argument name is totally debatable. Will also require a pull request in `tokenizers`. 
The goal is to be able to simply activate and de-activate the special token splitting. Feature was asked in #22490, and is required for some production type cases, where users pass inputs and we don't want them to be able to hack them
[`split_special_tokens`] Add support for `split_special_tokens` argument to encode. # What does this PR do?
Argument name is totally debatable. Will also require a pull request in `tokenizers`. 
The goal is to be able to simply activate and de-activate the special token splitting. Feature was asked in #22490, and is required for some production type cases, where users pass inputs and we don't want them to be able to hack them
#!/usr/bin/env bash
python run_common_voice.py \
    --model_name_or_path="facebook/wav2vec2-large-xlsr-53" \
    --dataset_config_name="tr" \
    --output_dir=./wav2vec2-large-xlsr-turkish-demo \
    --overwrite_output_dir \
    --num_train_epochs="5" \
    --per_device_train_batch_size="16" \
    --evaluation_strategy="steps" \
    --learning_rate="3e-4" \
    --warmup_steps="500" \
    --fp16 \
    --freeze_feature_extractor \
    --save_steps="400" \
    --eval_steps="400" \
    --save_total_limit="3" \
    --logging_steps="400" \
    --group_by_length \
    --feat_proj_dropout="0.0" \
    --layerdrop="0.1" \
    --gradient_checkpointing \
    --do_train --do_eval

#!/usr/bin/env bash
python run_common_voice.py \
    --model_name_or_path="facebook/wav2vec2-large-xlsr-53" \
    --dataset_config_name="tr" \
    --output_dir=./wav2vec2-large-xlsr-turkish-demo \
    --overwrite_output_dir \
    --num_train_epochs="5" \
    --per_device_train_batch_size="16" \
    --evaluation_strategy="steps" \
    --learning_rate="3e-4" \
    --warmup_steps="500" \
    --fp16 \
    --freeze_feature_extractor \
    --save_steps="400" \
    --eval_steps="400" \
    --save_total_limit="3" \
    --logging_steps="400" \
    --group_by_length \
    --feat_proj_dropout="0.0" \
    --layerdrop="0.1" \
    --gradient_checkpointing \
    --do_train --do_eval
