<jupyter_start><jupyter_text>Login HuggingFace to Upload Model<jupyter_code># !git config --global credential.helper store

# from huggingface_hub import notebook_login
# notebook_login()<jupyter_output><empty_output><jupyter_text>1. Training<jupyter_code>import torch
torch.cuda.empty_cache()
!nvidia-smi
def print_encoded(encoded_inputs):
    for ts in encoded_inputs:
        print(ts, encoded_inputs[ts])<jupyter_output><empty_output><jupyter_text>1.1 Prepare-Datasets<jupyter_code>import transformers
print(transformers.__version__)
model_checkpoint = "Helsinki-NLP/opus-mt-en-mul"
from datasets import load_dataset, load_metric, DatasetDict, Dataset

# raw_datasets = load_dataset("scb_mt_enth_2020", name="then")
metric = load_metric("sacrebleu")
import pandas as pd
def pre_process_from_csv(path, n_row=100000):
    df_5M = pd.read_csv(path, nrows=n_row)
    list_5M = df_5M.to_dict('records')
    list_sub = ['LST_Corpus']*len(list_5M)
    dict_5M = pd.DataFrame({"translation": list_5M, "subdataset": list_sub})
    return dict_5M
raw_datasets = pre_process_from_csv('TEST_dataset_5M.csv', 1000000) # 1Mill to the top
raw_datasets['translation']
cut_datasets = DatasetDict()
cut_datasets = Dataset.from_pandas(raw_datasets, split="train+validation").train_test_split(0.1)
cut_datasets
import datasets, random
import pandas as pd
from IPython.display import display, HTML

def show_random_element(dataset, num_examples=5):
    assert num_examples <= len(dataset), "Can't pick more elements than dataset"
    picks = []
    for n_ in range(num_examples):
        pick = random.randint(0, len(dataset)-1)
        while pick in picks: # random inside
            pick = random.randint(0, len(dataset)-1)
        picks.append(pick)

    df = pd.DataFrame(dataset[picks])
    for column, typ in dataset.features.items():
        if isinstance(typ, datasets.ClassLabel):
            df[column] = df[column].transform(lambda i: typ.names[i])
    display(HTML(df.to_html()))
show_random_element(cut_datasets['train'])
cut_datasets['train'][0:2]<jupyter_output><empty_output><jupyter_text>Metric<jupyter_code># metric
metric = load_metric("sacrebleu")
fake_preds = ["hello there", "general kenobi"]
fake_labels = [["hello there"], ["general kenobi"]]
metric.compute(predictions=fake_preds, references=fake_labels)<jupyter_output><empty_output><jupyter_text>1.2 Preprocessing the Data<jupyter_code>from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
# Set Source and Target Language

if 'mbart' in model_checkpoint:
    tokenizer.src_lang = 'th'
    tokenizer.tgt_lang = 'en'
if model_checkpoint in ["t5-small", "t5-base", "t5-larg", "t5-3b", "t5-11b"]:
    prefix = "translate Thai to English: "
elif model_checkpoint in ["Helsinki-NLP/opus-mt-en-mul"]:
    prefix = '>>tha<<'
else:
    prefix = ""
prefix
max_input_len  = 128
max_target_len = 128

source_lang = 'en'
target_lang = 'th'

def preprocess_function(examples):
#     inputs = [prefix + ex[source_lang] for ex in examples['translation']]
    inputs, targets = [], []
    for ex_ in (examples['translation']):
        ex_target = ex_[target_lang]
        # SKIP NULL THING
        if ex_target is not None and ex_[source_lang] is not None:
            targets.append(ex_[target_lang])
            inputs.append(prefix + ex_[source_lang])
            
    model_inputs = tokenizer(inputs, max_length=max_input_len, truncation=True) # Pad to longest word (128 char)
    
    with tokenizer.as_target_tokenizer():
        labels = tokenizer(targets, max_length=max_target_len, truncation=True)
    
    model_inputs['labels'] = labels['input_ids']
    return model_inputs
tokenized_datasets = cut_datasets.map(preprocess_function, 
                                      batched=True, 
                                      batch_size=1000, 
                                      num_proc=32, 
                                      remove_columns=["translation", "subdataset"], # Prevent ArrowInvalid Error (Skip)
                                     )
tokenized_datasets
tokenized_datasets['train'][100]<jupyter_output><empty_output><jupyter_text>1.3 Fine-tuning the model<jupyter_code>from transformers import AutoModelForSeq2SeqLM, DataCollatorForSeq2Seq, Seq2SeqTrainingArguments, Seq2SeqTrainer

model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)
model_checkpoint
torch.cuda.empty_cache()
!nvidia-smi
model_name = model_checkpoint.split('/')[-1]
new_model_name = f'{model_name}-finetuned-1M-{source_lang}-to-{target_lang}'
print('New Model Name')
print(new_model_name)
batch_size = 32
args = Seq2SeqTrainingArguments(
    new_model_name,
    evaluation_strategy = "epoch",
    learning_rate = 2e-5,
    per_device_train_batch_size = batch_size,
    per_device_eval_batch_size  = batch_size,
#     gradient_checkpointing=True, # For Accelerate Training
    weight_decay =0.01,
    save_total_limit = 3,
    num_train_epochs=1,
    predict_with_generate=True,
    dataloader_num_workers=32, # Multi-tread CPU
    fp16=True,
    eval_accumulation_steps = 10, # Reduce Using GPU Ram When Evaulation
    push_to_hub = False,
)<jupyter_output><empty_output><jupyter_text>Create Data collator<jupyter_code>data_collator = DataCollatorForSeq2Seq(tokenizer, model = model)
import numpy as np

def postprocess_text(preds, labels):
    preds = [pred.strip() for pred in preds]
    labels= [[label.strip()] for label in labels]
    return preds, labels

def compute_metrics(eval_preds):
    preds, labels = eval_preds
    if isinstance(preds, tuple):
        preds = preds[0]
    print('preds', preds)
    print('labels', labels)
    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens = True)

    # Replace -100 in the labels as we can't decode them.
    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)

    # Some simple post-processing
    decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels)
    
    result = metric.compute(predictions=decoded_preds, references=decoded_labels)
    result = {"bleu": result['score']}

    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds]
    result["gen_len"] = np.mean(prediction_lens)
    result = {k: round(v, 4) for k, v in result.items()}
    return result
trainer = Seq2SeqTrainer(
    model,
    args,
    train_dataset= tokenized_datasets['train'],
#     eval_dataset = tokenized_datasets['test'],
    eval_dataset = None,
    data_collator = data_collator,
    tokenizer = tokenizer,
#     compute_metrics = compute_metrics,
)<jupyter_output>Using amp half precision backend<jupyter_text>1.4 *train time!*<jupyter_code>trainer.train()
trainer.save_model()<jupyter_output>Saving model checkpoint to opus-mt-en-mul-finetuned-1M-en-to-th
Configuration saved in opus-mt-en-mul-finetuned-1M-en-to-th/config.json
Model weights saved in opus-mt-en-mul-finetuned-1M-en-to-th/pytorch_model.bin
tokenizer config file saved in opus-mt-en-mul-finetuned-1M-en-to-th/tokenizer_config.json
Special tokens file saved in opus-mt-en-mul-finetuned-1M-en-to-th/special_tokens_map.json<jupyter_text>When Loading the model``` from transformers import AutoModelForSeq2SeqLMmodel = AutoModelForSeq2SeqLM.from_pretrained("sgugger/my-awesome-model")```  2. Using Translation Model  Using AutoModel (With model selection)<jupyter_code>?AutoModelForSeq2SeqLM.
# metric = load_metric("bleu")
# fake_preds = [["hello"],["there"], ["general"], ['kenobi']]
# fake_labels = [["hello"],["there"], ["general"], ['kenobi']]
# # fake_labels = [["hello there"], ["general kenobi"]]
# metric.compute(predictions=fake_preds, 
#                references=fake_labels)
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
model_pretrain = "Helsinki-NLP/opus-mt-en-mul"
new_model_name = 'TNANA-attacut-th-to-en-pt2/checkpoint-23000'
model_for_predict = f"./{new_model_name}"
# model_for_predict = "huak95/opus-mt-th-en-finetuned-5k-th-to-en"
print(model_for_predict)
model_pt = AutoModelForSeq2SeqLM.from_pretrained(model_pretrain)
model = AutoModelForSeq2SeqLM.from_pretrained(model_for_predict)
tokenizer = AutoTokenizer.from_pretrained(model_for_predict)
from transformers import Seq2SeqTrainer
trainer = Seq2SeqTrainer(
    model,
    tokenizer = tokenizer,
)
!git lfs track 
model.push_to_hub('huak95/TNANA-attacut-th-to-en')<jupyter_output>Configuration saved in huak95/TNANA-attacut-th-to-en/config.json
Model weights saved in huak95/TNANA-attacut-th-to-en/pytorch_model.bin<jupyter_text>PUSH TO HUB >_<<jupyter_code>!git lfs track 

# model.push_to_hub('huak95/opus-mt-en-mul-finetuned-1M-en-to-th')
tokenizer.push_to_hub('huak95/opus-mt-en-mul-finetuned-1M-en-to-th')<jupyter_output>Listing tracked patterns
    opus-mt-th-en-finetuned-5k-th-to-en/* (.gitattributes)
Listing excluded patterns<jupyter_text>Testing >0<<jupyter_code>def harry_translate(input_texts, model=model_pt):
    inputs = tokenizer('>>tha<<' + input_texts['en'], return_tensors = "pt")
    
    outputs = model.generate(inputs["input_ids"],
                             max_length=40,
                             num_beams=4,
                             early_stopping=True)

    return tokenizer.decode(outputs[0]).replace('<pad> ',''), input_texts['th']
datasets_size = len(cut_datasets['test']['translation'])
rand_idx = np.random.randint(datasets_size, size=10)

for i in rand_idx:
    print(f'index: {i}')

    text_ = cut_datasets['test']['translation'][i]
    thai_text = text_['en']
    pred_text_pt, real_text = harry_translate(text_, model_pt)
    pred_text, real_text = harry_translate(text_, model)
    
#     m_score = compute_metrics.compute(predictions=[pred_text], references=[real_text])

    print('thai_text ตัวอย่าง:\t',  thai_text)
    print('real_text เฉลย  :\t', real_text)
    print('pre_train ก่อนจูน :\t', pred_text_pt)
    print('pred_text หลังจูน :\t', pred_text)
    print()<jupyter_output>index: 83104
thai_text ตัวอย่าง:	 grunt
real_text เฉลย  :	 เนื้อ สุกร
pre_train ก่อนจูน :	 กรอนท์
pred_text หลังจูน :	 กรุนต์

index: 53707
thai_text ตัวอย่าง:	 the party gonna be great
real_text เฉลย  :	 งาน ปาร์ตี้ จะ ต้อง เยี่ยม
pre_train ก่อนจูน :	 ปาร์ตี้จะดี
pred_text หลังจูน :	 งานเลี้ยง จะ เยี่ยม

index: 85305
thai_text ตัวอย่าง:	 This is no small problem , since making a hole in the wrong place can cause excessive bleeding .
real_text เฉลย  :	 นอก ​ จาก ​ นั้น นัก ​ เจาะ ​ หลาย ​ คน ​ ขาด ​ ความ ​ เข้าใจ ​ เรื่อง ​ กายวิภาคศาสตร์ .
pre_train ก่อนจูน :	 นี่ไม่ใช่ปัญหาเล็ก เนื่องจากการทําลายในสถานที่ผิด สามารถทําให้เลือด
pred_text หลังจูน :	 นี่ ไม่ ใช่ ปัญหา เล็ก เนื่อง จาก การ ทํา หลุม ใน ที่ ผิด อาจ ทํา

index: 28693
thai_text ตัวอย่าง:	 Likewise , for us to become spiritually sound , we need to “ chew ” on God’s utterances so that these can be assimilated into our mind and heart .
real_text เฉลย  :	 นอก ​ จาก ​ นี้ ​ ก็ ​ ต้อง ​ มี ​ การ ​ เคี้ยว ​ อาหาร ​ ให้ ​ ละเอียด ​ แล[...]<jupyter_text>VIEW NAN DATA<jupyter_code>text_df = pd.DataFrame(cut_datasets['test']['translation'])

datasets_size = len(cut_datasets['test']['translation'])
rand_idx = np.random.randint(datasets_size, size=100)

datasets_small = text_df[text_df.en.isnull()]
for i in (datasets_small).index:
    print(f'index: {i}')


    thai_text = datasets_small['th'].loc[i]
    real_text = datasets_small['en'].loc[i]
#     pred_text_pt, real_text = harry_translate(text_, model_pt)
#     pred_text, real_text = harry_translate(text_, model)
    
#     m_score = compute_metrics.compute(predictions=[pred_text], references=[real_text])

    print('thai_text ตัวอย่าง:\t',  thai_text)
    print('real_text เฉลย  :\t', real_text)
#     print('pre_train ก่อนจูน :\t', pred_text_pt)
#     print('pred_text หลังจูน :\t', pred_text)
    print()
datasets_small['len_th'] = datasets_small['th']
datasets_small['th']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1227687924243694, 'fraction_numerical': 0.014162406771138858, 'mean_word_length': 4.574030826716488, 'pattern_counts': {'":': 3, '<': 52, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '222928', 'license': 'Apache-2.0', 'n_tokens_mistral': 4627, 'n_tokens_neox': 4522, 'n_words': 832, 'path': 'Train_en-th_harryy.ipynb', 'repo_name': 'huak95/machine-translation', 'size': '11933'}