<jupyter_start><jupyter_text>Cross Language Search We are using [this nice dataset](https://github.com/BangBOOM/Classical-Chinese)  Imports<jupyter_code># !pip install -Uqq git+https://github.com/raynardj/forgebox
from forgebox.imports import *
from forgebox.thunder.callbacks import DataFrameMetricsCallback
from forgebox.multiproc import DataFrameRowling
from gc_utils.env import *
from datasets import load_dataset
# from fastai.text.all import *
from unpackai.nlp import *
from tqdm.notebook import tqdm
import random
import pytorch_lightning as pl
import re

def remove_all_punkt(text):
    """
    Removes all punctuation from Chinese text.

    :param text: text to remove punctuation from
    :return: text with no punctuation
    """
    return re.sub(r'[^\w\s]', '', text)
remove_all_punkt("亳州水军千户胡进等领骑兵渡淝水，逾荆山，与宋兵战，杀获甚众，赏钞币有差。")<jupyter_output><empty_output><jupyter_text>Config<jupyter_code>DATA = Path(sys_loc('DATA')/"nlp"/"zh"/"cc_vs_zh")
TO_CLASSICAL = False<jupyter_output><empty_output><jupyter_text>Download data  Data  Combine data<jupyter_code>all_file = list(DATA.rglob("data/*"))
def open_file_to_lines(file):
    with open(file) as f:
        lines = f.read().splitlines()
    return lines

def pairing_the_file(files,kw):
    pairs = []
    for file in files:
        if kw not in file.name:
            file1 = file
            file2 = f"{file}{kw}"
            pairs.append((file1,file2))
    return pairs
pairs = pairing_the_file(all_file,"翻译")
def open_pairs(pairs):
    chunks = []
    for pair in tqdm(pairs, leave=False):
        file1,file2 = pair
        lines1 = open_file_to_lines(file1)
        lines2 = open_file_to_lines(file2)
        chunks.append(pd.DataFrame({"classical":lines1,"modern":lines2}))
    return pd.concat(chunks).sample(frac=1.).reset_index(drop=True)
data_df = open_pairs(pairs)
df = data_df.rename(
    columns = dict(
        zip(["modern","classical"],
             ["source","target"] if TO_CLASSICAL else ["target","source",]))
)
df.head()<jupyter_output><empty_output><jupyter_text>Loading tokenizer<jupyter_code>from transformers import (
    AutoTokenizer,
    AutoModelForMaskedLM,
    AutoModel,
    EncoderDecoderModel
    )
PRETRAINED = "bert-base-chinese"

tokenizer = AutoTokenizer.from_pretrained(PRETRAINED)<jupyter_output><empty_output><jupyter_text>Pytoch Dataset<jupyter_code>import random

def combine_randomly(data):
    if random.random()>.5:
        a,b = data['source'],data['target']
    else:
        a,b = data['target'],data['source']
    return f"{a}{b}"

def pick_randomly(data):
    return list(data.values())[int(random.random()>.5)]

def mixup(data):
    if len(data['target'])> 70:
        th = .7
    else:
        th = .3
    if random.random()>th:
        return combine_randomly(data)
    else:
        return pick_randomly(data)
class XLSearch(Dataset):
    def __init__(
        self, df, tokenizer,
        max_len=128,
        no_punkt:bool = False,
        mlm_probability:float = .15,
    ):
        """
        no_punkt, do we ramdomly remove punctuation
        from source sentence
        """
        super().__init__()
        self.df = df.reset_index(drop=True)
        self.tokenizer = tokenizer
        self.max_len = max_len
        self.mlm_probability = mlm_probability
        
    def __len__(self, ):
        return len(self.df)

    def __getitem__(self, idx):
        return mixup(dict(self.df.loc[idx]))

    def collate(self, data):
        inputs = self.tokenizer(
            list(data),
            max_length=self.max_len,
            padding='max_length',
            truncation=True,
            return_tensors='pt',
        )
        return self.mlm_masking(inputs)
    
    def mlm_masking(self,inputs):
        """
        convert inputs for masked language modeling
        """
        if self.mlm_probability is None:
            return inputs
        input_ids = inputs.input_ids
        token_type_ids = inputs.token_type_ids
        
        # masking input_ids
        masked = input_ids.clone()
        masked[
            torch.rand(input_ids.shape).to(input_ids.device) < self.mlm_probability
        ] = self.tokenizer.mask_token_id
        
        labels = input_ids.clone()
        labels[token_type_ids == 1] = -100
        labels[labels==0] = -100
        token_type_ids[masked==self.tokenizer.mask_token_id] = 1
        labels[token_type_ids == 0] = -100
        
        inputs['input_ids'] = masked
        inputs['labels'] = labels
        inputs['token_type_ids'] = token_type_ids
        return inputs

    def dataloader(self, batch_size, shuffle=True):
        return DataLoader(
            self,
            batch_size=batch_size,
            shuffle=shuffle,
            collate_fn=self.collate,
        )

    def split_train_valid(self, valid_size=0.1):
        split_index = int(len(self) * (1 - valid_size))
        cls = type(self)
        shuffled = self.df.sample(frac=1).reset_index(drop=True)
        train_set = cls(
            shuffled.iloc[:split_index],
            tokenizer=self.tokenizer,
            max_len=self.max_len,
        )
        valid_set = cls(
            shuffled.iloc[split_index:],
            tokenizer=self.tokenizer,
            max_len=self.max_len,
        )
        return train_set, valid_set
ds = XLSearch(df, tokenizer, )
ds[5]<jupyter_output><empty_output><jupyter_text>Different ways of mixing and masking  PL datamodule<jupyter_code>class DataModule(pl.LightningDataModule):
    def __init__(
        self, df,
        tokenizer,
        batch_size=12,
        max_len=128,
        no_punkt:bool=False):
        super().__init__()
        self.df = df
        self.ds = XLSearch(df,
                          tokenizer,
                          max_len=max_len,)
        self.tokenizer = tokenizer
        self.max_len = max_len
        self.batch_size = batch_size

    def setup(self, stage=None):
        self.train_set, self.valid_set = self.ds.split_train_valid()

    def train_dataloader(self):
        return self.train_set.dataloader(
            batch_size=self.batch_size, shuffle=True)

    def val_dataloader(self):
        return self.valid_set.dataloader(
            batch_size=self.batch_size*2, shuffle=False)
data_module = DataModule(
    df, tokenizer,
    batch_size=64,
    max_len=256,
    no_punkt=False if TO_CLASSICAL else True,)
data_module.setup()
inputs = next(iter(data_module.train_dataloader()))
inputs<jupyter_output><empty_output><jupyter_text>if we are doing clasical Chinese to modern Chinese, we can randomly set half of the input without any punctuation, as many data source might be<jupyter_code># tokenizer.batch_decode(
#     inputs.input_ids,skip_special_tokens=False
# )<jupyter_output><empty_output><jupyter_text>Load pretrained models  Model<jupyter_code># loading pretrained model
model = AutoModelForMaskedLM.from_pretrained(PRETRAINED
)
class MaskedLM(pl.LightningModule):
    def __init__(
        self,
        model):
        super().__init__()
        self.model = model

    def forward(self, **kwargs):
        return self.model(**kwargs)

    def accuracy(self, batch_input, outputs):
        """
        Accuracy for masked language model
        """
        mask_mask = batch_input.labels != -100
        predictions = outputs.logits.argmax(-1)[mask_mask]
        targets = batch_input.labels[mask_mask]
        return (predictions == targets).float().mean()

    def training_step(self, batch, batch_idx):
        inputs = dict(
            input_ids=batch.input_ids,
            attention_mask=batch.attention_mask,
            labels=batch.labels,
            )
        outputs = self(**inputs)
        self.log("loss", outputs.loss, prog_bar=True)
        self.log("acc",
            self.accuracy(batch, outputs),
            on_step=True, prog_bar=True)
        return outputs.loss

    def validation_step(self, batch, batch_idx):
        inputs = dict(
            input_ids=batch.input_ids,
            attention_mask=batch.attention_mask,
            labels=batch.labels,
            )
        outputs = self(**inputs)
        self.log("val_loss", outputs.loss, prog_bar=True)
        self.log("val_acc",
            self.accuracy(batch, outputs),
            on_step=False, prog_bar=True)
        return outputs.loss
    
    def configure_optimizers(self):
        return torch.optim.Adam(self.parameters(), lr=1e-6)
module = MaskedLM(model)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>TASK = "xlsearch_cc_zh"
tb_logger = pl.loggers.TensorBoardLogger(
    save_dir=f"/GCI/tensorboard/{TASK}",
    name=TASK,
    )

save_cb = pl.callbacks.ModelCheckpoint(
    dirpath=f"/GCI/transformers/weights/{TASK}",
    save_top_k=3,
    verbose=True,
    monitor="acc",
    save_weights_only=True,
    every_n_train_steps=1024,
    mode="max",
    )

trainer = pl.Trainer(
    gpus=[1,],
    max_epochs=10,
    logger = [tb_logger,],
    callbacks=[save_cb,
#                DataFrameMetricsCallback()
              ],
    )

trainer.fit(
    module,
    datamodule = data_module
    )<jupyter_output>GPU available: True, used: True
TPU available: False, using: 0 TPU cores
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]

  | Name  | Type            | Params
------------------------------------------
0 | model | BertForMaskedLM | 102 M 
------------------------------------------
102 M     Trainable params
0         Non-trainable params
102 M     Total params
409.161   Total estimated model params size (MB)<jupyter_text>Save<jupyter_code>best = save.best
module.load_state_dict(torch.load(best, map_location="cpu")['state_dict'])
# encoder_decoder.push_to_hub("raynardj/wenyanwen-chinese-translate-to-ancient")
# encoder_tokenizer.push_to_hub("raynardj/wenyanwen-chinese-translate-to-ancient")<jupyter_output>To https://user:eOwfuFZJHbcMgbzVtVPDaSGtpbpjumsgTzZtfKlrMbSECzypnCYHZGDhHVsHRsYZzvdrkcxbnnSXRROfqdNRYfMvVfaVSOTxORkEUcMnAPEWXhkWpVEDrgfUZJdmleTx@huggingface.co/raynardj/wenyanwen-chinese-translate-to-ancient
   5ee2133..ab72fa4  main -> main
{'fraction_non_alphanumeric': 0.11651549885378251, 'fraction_numerical': 0.00956842420013954, 'mean_word_length': 2.7835595776772246, 'pattern_counts': {'":': 2, '<': 40, '<?xml version=': 0, '>': 44, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '300773', 'license': 'MIT', 'n_tokens_mistral': 3449, 'n_tokens_neox': 3367, 'n_words': 580, 'path': 'nbs/xlsearch.ipynb', 'repo_name': 'raynardj/yuan', 'size': '10033'}