Add application file

Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.9-slim
+
+WORKDIR /code
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential gcc \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir Cython
+
+COPY ./requirements.txt /code/
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY ./multi_lingual.py ./urdu_punkt.py ./main.py /code/
+COPY ./models/ /code/models/
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

main.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+import gradio as gr
+from fastapi import FastAPI
+from urdu_punkt import Urdu
+from multi_lingual import MultiLingual
+from langdetect import detect, DetectorFactory
+
+CUSTOM_PATH = "/punctuate"
+DetectorFactory.seed = 42
+
+app = FastAPI()
+nemo_model = Urdu()
+multi_model = MultiLingual()
+
+
+def punctuate(text: str) -> str:
+    if detect(text) == "ur":
+        return nemo_model.punctuate(text)
+    else:
+        return multi_model.punctuate(text)
+
+
+title = "SELMA H2020 — Multilingual Punctuation & Casing Prediction"
+description = "Supported languages are: Amharic, Bengali, German, English, Spanish, French, Hindi, Italian, Latvian, Pashto, Portuguese, Russian, Tamil and Urdu."
+article = "<p style='text-align: center'><a href='https://selma-project.eu' target='_blank'>SELMA-H2020</a></p>"
+
+text_input = gr.Textbox(label="Enter some text")
+result_output = gr.Textbox(label="Result")
+
+io = gr.Interface(
+    fn=punctuate,
+    title=title,
+    description=description,
+    article=article,
+    theme=gr.themes.Soft(),
+    inputs=text_input,
+    outputs=result_output,
+    allow_flagging="never",
+    css="footer {visibility: hidden}",
+)
+
+app = gr.mount_gradio_app(app, io, path=CUSTOM_PATH)

models/multilingual/config.yaml

@@ -0,0 +1,43 @@
+# am: Amharic
+# bn: Bengali
+# de: German
+# en: English
+# es: Spanish
+# fr: French
+# hi: Hindi
+# it: Italian
+# lv: Latvian
+# ps: Pashto
+# pt: Portuguese
+# ru: Russian
+# ta: Tamil
+
+languages:  ["am", "bn", "de", "en", "es", "fr", "hi", "it", "lv", "ps", "pt", "ru", "ta"]
+
+max_length: 256
+
+# just for Spanish
+pre_labels: [
+  "<NULL>",
+  "¿",
+]
+
+post_labels: [
+  "<NULL>",
+  "<ACRONYM>",
+  ".",
+  ",",
+  "?",
+  "？",
+  "，",
+  "。",
+  "、",
+  "・",
+  "।",
+  "؟",
+  "،",
+  ";",
+  "።",
+  "፣",
+  "፧",
+]

models/multilingual/nemo_model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c43ca686dabc237c3b06be834b9423c07580fef7e2b1a6c09976f7d60caa5d89
+size 1112481438

models/multilingual/xlm_roberta_encoding.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f944d0be93b275f62e1913fd409f378ddbba108e57fe4a9cb47e8c047f6bef1
+size 5069059

models/urdu/config.json

@@ -0,0 +1,43 @@
+{
+  "_name_or_path": "bert-base-multilingual-cased",
+  "architectures": [
+    "BertForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "directionality": "bidi",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "O",
+    "1": "F",
+    "2": "C",
+    "3": "Q"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "C": 2,
+    "F": 1,
+    "O": 0,
+    "Q": 3
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_fc_size": 768,
+  "pooler_num_attention_heads": 12,
+  "pooler_num_fc_layers": 3,
+  "pooler_size_per_head": 128,
+  "pooler_type": "first_token_transform",
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.33.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 119547
+}

models/urdu/model_args.json

@@ -0,0 +1 @@
+{"adafactor_beta1": null, "adafactor_clip_threshold": 1.0, "adafactor_decay_rate": -0.8, "adafactor_eps": [1e-30, 0.001], "adafactor_relative_step": true, "adafactor_scale_parameter": true, "adafactor_warmup_init": true, "adam_betas": [0.9, 0.999], "adam_epsilon": 1e-08, "best_model_dir": "./titanen_outputs/best_model/", "cache_dir": "./titanen_cache/", "config": {}, "cosine_schedule_num_cycles": 0.5, "custom_layer_parameters": [], "custom_parameter_groups": [], "dataloader_num_workers": 0, "do_lower_case": false, "dynamic_quantize": false, "early_stopping_consider_epochs": false, "early_stopping_delta": 0, "early_stopping_metric": "eval_loss", "early_stopping_metric_minimize": true, "early_stopping_patience": 3, "encoding": null, "eval_batch_size": 8, "evaluate_during_training": true, "evaluate_during_training_silent": true, "evaluate_during_training_steps": 2000, "evaluate_during_training_verbose": true, "evaluate_each_epoch": true, "fp16": true, "gradient_accumulation_steps": 1, "learning_rate": 4e-05, "local_rank": -1, "logging_steps": 50, "loss_type": null, "loss_args": {}, "manual_seed": 42, "max_grad_norm": 1.0, "max_seq_length": 512, "model_name": "bert-base-multilingual-cased", "model_type": "bert", "multiprocessing_chunksize": -1, "n_gpu": 1, "no_cache": false, "no_save": false, "not_saved_args": [], "num_train_epochs": 3, "optimizer": "AdamW", "output_dir": "./titanen_outputs/", "overwrite_output_dir": true, "polynomial_decay_schedule_lr_end": 1e-07, "polynomial_decay_schedule_power": 1.0, "process_count": 18, "quantized_model": false, "reprocess_input_data": false, "save_best_model": true, "save_eval_checkpoints": false, "save_model_every_epoch": true, "save_optimizer_and_scheduler": true, "save_steps": -1, "scheduler": "linear_schedule_with_warmup", "silent": false, "skip_special_tokens": true, "tensorboard_dir": null, "thread_count": null, "tokenizer_name": null, "tokenizer_type": null, "train_batch_size": 8, "train_custom_parameters_only": false, "use_cached_eval_features": false, "use_early_stopping": false, "use_hf_datasets": false, "use_multiprocessing": true, "use_multiprocessing_for_evaluation": true, "wandb_kwargs": {"name": "bert-base-multilingual-titanen", "entity": "tugtekin", "notes": "Training punctuation prediction using BERT.", "tags": ["urdu", "bert", "punctuation"]}, "wandb_project": "urdu-punctuation", "warmup_ratio": 0.06, "warmup_steps": 30073, "weight_decay": 0.0, "model_class": "NERModel", "classification_report": false, "labels_list": ["O", "F", "C", "Q"], "lazy_loading": true, "lazy_loading_start_line": 0, "onnx": false, "special_tokens_list": []}

models/urdu/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:026c4bec45da24b0b7aac90ce21be9daa184910ac543f31df32ae34e9a9ce73b
+size 1418293317

models/urdu/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39fdcf468bb421bcfc6f9028c220032130f9e4d5f685b6c293d4c7484d41ff29
+size 709131433

models/urdu/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:091e952a51388e2a6e71cfa983884d27b6b07a02767ade8eb751f3348703458b
+size 627

models/urdu/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

models/urdu/tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": false,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

models/urdu/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf79804213bb807517d0519a6bcc18e290c8770eda02d4538f4a90507cfce545
+size 3259

multi_lingual.py

@@ -0,0 +1,310 @@
+#!/usr/bin/env python
+
+import os
+import torch
+import string
+import onnxruntime as ort
+from dataclasses import dataclass
+from omegaconf import OmegaConf
+from typing import List, Optional, Union, Dict
+from sentencepiece import SentencePieceProcessor
+from torch.utils.data import Dataset, DataLoader
+from typing import Iterator, List, Iterable, Tuple
+
+ACRONYM_TOKEN = "<ACRONYM>"
+torch.set_grad_enabled(False)
+torch.backends.cudnn.enabled = False
+os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+
+
+@dataclass
+class PunctCapConfigONNX:
+    spe_filename: str = "xlm_roberta_encoding.model"
+    model_filename: str = "nemo_model.onnx"
+    config_filename: str = "config.yaml"
+    directory: Optional[str] = None
+
+
+class PunctCapModelONNX:
+    def __init__(self, cfg: PunctCapConfigONNX):
+        self._spe_path = os.path.join(cfg.directory, cfg.spe_filename)
+        onnx_path = os.path.join(cfg.directory, cfg.model_filename)
+        config_path = os.path.join(cfg.directory, cfg.config_filename)
+
+        self._tokenizer: SentencePieceProcessor = SentencePieceProcessor(self._spe_path)
+        self._ort_session: ort.InferenceSession = ort.InferenceSession(onnx_path)
+        self._config = OmegaConf.load(config_path)
+        self._max_len = self._config.max_length
+        self._pre_labels: List[str] = self._config.pre_labels
+        self._post_labels: List[str] = self._config.post_labels
+        self._languages: List[str] = self._config.languages
+        self._null_token = self._config.get("null_token", "<NULL>")
+
+    def _setup_dataloader(self, texts: List[str], batch_size_tokens: int, overlap: int) -> DataLoader:
+        dataset: TextInferenceDataset = TextInferenceDataset(
+            texts=texts,
+            batch_size_tokens=batch_size_tokens,
+            overlap=overlap,
+            max_length=self._max_len,
+            spe_model_path=self._spe_path,
+        )
+        return DataLoader(
+            dataset=dataset,
+            collate_fn=dataset.collate_fn,
+            batch_sampler=dataset.sampler,
+        )
+
+    def punctuation_removal(self, texts: List[str]) -> List[str]:
+        punkt = string.punctuation + """`÷×؛<>_()*&^%][ـ،/:"؟.,'{}~¦+|！”…–ـ""" + """！？｡。"""
+        punkt = punkt.replace("-", "")
+        punkt = punkt.replace("'", "")
+        punkt += "„“"
+        return [text.translate(str.maketrans("", "", punkt)).lower().strip() for text in texts]
+
+    def infer(
+        self,
+        texts: List[str],
+        apply_sbd: bool = False,
+        batch_size_tokens: int = 4096,
+        overlap: int = 16,
+    ) -> Union[List[str], List[List[str]]]:
+        texts = self.punctuation_removal(texts)
+
+        collectors: List[PunctCapCollector] = [
+            PunctCapCollector(sp_model=self._tokenizer, apply_sbd=apply_sbd, overlap=overlap)
+            for _ in range(len(texts))
+        ]
+        dataloader: DataLoader = self._setup_dataloader(texts=texts, batch_size_tokens=batch_size_tokens, overlap=overlap)
+        for batch in dataloader:
+            input_ids, batch_indices, input_indices, lengths = batch
+            pre_preds, post_preds, cap_preds, seg_preds = self._ort_session.run(None, {"input_ids": input_ids.numpy()})
+            batch_size = input_ids.shape[0]
+            for i in range(batch_size):
+                length = lengths[i].item()
+                batch_idx = batch_indices[i].item()
+                input_idx = input_indices[i].item()
+                segment_ids = input_ids[i, 1 : length - 1].tolist()
+                segment_pre_preds = pre_preds[i, 1 : length - 1].tolist()
+                segment_post_preds = post_preds[i, 1 : length - 1].tolist()
+                segment_cap_preds = cap_preds[i, 1 : length - 1].tolist()
+                segment_sbd_preds = seg_preds[i, 1 : length - 1].tolist()
+                pre_tokens = [self._pre_labels[i] for i in segment_pre_preds]
+                post_tokens = [self._post_labels[i] for i in segment_post_preds]
+                pre_tokens = [x if x != self._null_token else None for x in pre_tokens]
+                post_tokens = [x if x != self._null_token else None for x in post_tokens]
+                collectors[batch_idx].collect(
+                    ids=segment_ids,
+                    pre_preds=pre_tokens,
+                    post_preds=post_tokens,
+                    cap_preds=segment_cap_preds,
+                    sbd_preds=segment_sbd_preds,
+                    idx=input_idx,
+                )
+        outputs: Union[List[str], List[List[str]]] = [x.produce() for x in collectors]
+        return outputs
+
+
+@dataclass
+class TokenizedSegment:
+    input_ids: List[int]
+    batch_idx: int
+    input_idx: int
+
+    def __len__(self) -> int:
+        return len(self.input_ids)
+
+
+class TokenBatchSampler(Iterable):
+    def __init__(self, segments: List[TokenizedSegment], batch_size_tokens: int):
+        self._batches = self._make_batches(segments, batch_size_tokens)
+
+    def _make_batches(self, segments: List[TokenizedSegment], batch_size_tokens: int) -> List[List[int]]:
+        segments_with_index = [(segment, i) for i, segment in enumerate(segments)]
+        segments_with_index.sort(key=lambda x: len(x[0]), reverse=True)
+
+        batches, current_batch_elements, current_max_len = [], [], 0
+
+        for segment, idx in segments_with_index:
+            potential_max_len = max(current_max_len, len(segment))
+
+            if potential_max_len * (len(current_batch_elements) + 1) > batch_size_tokens:
+                batches.append(current_batch_elements)
+                current_batch_elements, current_max_len = [], 0
+
+            current_batch_elements.append(idx)
+            current_max_len = potential_max_len
+
+        if current_batch_elements:
+            batches.append(current_batch_elements)
+
+        return batches
+
+    def __iter__(self) -> Iterator:
+        yield from self._batches
+
+    def __len__(self) -> int:
+        return len(self._batches)
+
+
+class TextInferenceDataset(Dataset):
+    def __init__(
+        self,
+        texts: List[str],
+        spe_model_path: str,
+        batch_size_tokens: int = 4096,
+        max_length: int = 512,
+        overlap: int = 32,
+    ):
+        self._spe_model = SentencePieceProcessor(spe_model_path)
+        self._segments = self._tokenize_inputs(texts, max_length, overlap)
+        self._sampler = TokenBatchSampler(self._segments, batch_size_tokens)
+
+    @property
+    def sampler(self) -> Iterable:
+        return self._sampler
+
+    def _tokenize_inputs(self, texts: List[str], max_len: int, overlap: int) -> List[TokenizedSegment]:
+        max_len -= 2
+        segments = []
+
+        for batch_idx, text in enumerate(texts):
+            ids, start, input_idx = self._spe_model.EncodeAsIds(text), 0, 0
+
+            while start < len(ids):
+                adjusted_start = start - overlap if input_idx else 0
+                segments.append(
+                    TokenizedSegment(
+                        ids[adjusted_start : adjusted_start + max_len],
+                        batch_idx,
+                        input_idx,
+                    )
+                )
+                start += max_len - overlap
+                input_idx += 1
+
+        return segments
+
+    def __len__(self) -> int:
+        return len(self._segments)
+
+    def __getitem__(self, idx: int) -> Tuple[torch.Tensor, int, int]:
+        segment = self._segments[idx]
+        input_ids = torch.Tensor([self._spe_model.bos_id(), *segment.input_ids, self._spe_model.eos_id()])
+        return input_ids, segment.batch_idx, segment.input_idx
+
+    def collate_fn(self, batch: List[Tuple[torch.Tensor, int, int]]) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+        input_ids = [x[0] for x in batch]
+        lengths = torch.tensor([x.shape[0] for x in input_ids])
+        max_len = lengths.max().item()
+
+        batched_ids = torch.full((len(input_ids), max_len), self._spe_model.pad_id())
+        for idx, ids in enumerate(input_ids):
+            batched_ids[idx, : lengths[idx]] = ids
+
+        return (
+            batched_ids,
+            torch.tensor([x[1] for x in batch]),
+            torch.tensor([x[2] for x in batch]),
+            lengths,
+        )
+
+
+@dataclass
+class PCSegment:
+    ids: List[int]
+    pre_preds: List[Optional[str]]
+    post_preds: List[Optional[str]]
+    cap_preds: List[List[int]]
+    sbd_preds: List[int]
+
+    def __len__(self):
+        return len(self.ids)
+
+
+class PunctCapCollector:
+    def __init__(self, apply_sbd: bool, overlap: int, sp_model: SentencePieceProcessor):
+        self._segments: Dict[int, PCSegment] = {}
+        self._apply_sbd = apply_sbd
+        self._overlap = overlap
+        self._sp_model = sp_model
+
+    def collect(
+        self,
+        ids: List[int],
+        pre_preds: List[Optional[str]],
+        post_preds: List[Optional[str]],
+        sbd_preds: List[int],
+        cap_preds: List[List[int]],
+        idx: int,
+    ):
+        self._segments[idx] = PCSegment(
+            ids=ids,
+            pre_preds=pre_preds,
+            post_preds=post_preds,
+            sbd_preds=sbd_preds,
+            cap_preds=cap_preds,
+        )
+
+    def produce(self) -> Union[List[str], str]:
+        ids: List[int] = []
+        pre_preds: List[Optional[str]] = []
+        post_preds: List[Optional[str]] = []
+        cap_preds: List[List[int]] = []
+        sbd_preds: List[int] = []
+
+        for i in range(len(self._segments)):
+            segment = self._segments[i]
+            start = 0
+            stop = len(segment)
+            if i > 0:
+                start += self._overlap // 2
+            if i < len(self._segments) - 1:
+                stop -= self._overlap // 2
+
+            ids.extend(segment.ids[start:stop])
+            pre_preds.extend(segment.pre_preds[start:stop])
+            post_preds.extend(segment.post_preds[start:stop])
+            sbd_preds.extend(segment.sbd_preds[start:stop])
+            cap_preds.extend(segment.cap_preds[start:stop])
+
+        input_tokens = [self._sp_model.IdToPiece(x) for x in ids]
+        output_texts: List[str] = []
+        current_chars: List[str] = []
+
+        for token_idx, token in enumerate(input_tokens):
+            if token.startswith("▁") and current_chars:
+                current_chars.append(" ")
+            char_start = 1 if token.startswith("▁") else 0
+
+            for token_char_idx, char in enumerate(token[char_start:], start=char_start):
+                if token_char_idx == char_start and pre_preds[token_idx] is not None:
+                    current_chars.append(pre_preds[token_idx])
+                if cap_preds[token_idx][token_char_idx]:
+                    char = char.upper()
+                current_chars.append(char)
+
+                label = post_preds[token_idx]
+                if label == ACRONYM_TOKEN:
+                    current_chars.append(".")
+                elif token_char_idx == len(token) - 1 and post_preds[token_idx] is not None:
+                    current_chars.append(post_preds[token_idx])
+                if self._apply_sbd and token_char_idx == len(token) - 1 and sbd_preds[token_idx]:
+                    output_texts.append("".join(current_chars))
+                    current_chars = []
+
+        if current_chars:
+            output_texts.append("".join(current_chars))
+        if not self._apply_sbd:
+            if len(output_texts) > 1:
+                raise ValueError(f"Not applying SBD but got more than one result: {output_texts}")
+            return output_texts[0]
+        return output_texts
+
+
+class MultiLingual:
+    def __init__(self):
+        cfg = PunctCapConfigONNX(directory="/code/models/multilingual")
+        self._punctuator = PunctCapModelONNX(cfg)
+
+    def punctuate(self, data: str) -> str:
+        return self._punctuator.infer([data])[0]

requirements.txt

@@ -0,0 +1,11 @@
+--extra-index-url https://download.pytorch.org/whl/cpu
+fastapi==0.103.1
+gradio==3.44.3
+langdetect==1.0.9
+onnxruntime==1.15.1
+omegaconf==2.3.0
+pandas==2.1.0
+six==1.16.0
+simpletransformers==0.64.3
+tensorflow-datasets==4.9.3
+torch==1.13.1+cpu

urdu_punkt.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+
+import os
+import re
+import string
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
+os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0"
+
+from simpletransformers.ner import NERModel
+
+
+class BERTmodel:
+    def __init__(self, normalization="full", wrds_per_pred=256):
+        self.normalization = normalization
+        self.wrds_per_pred = wrds_per_pred
+        self.overlap_wrds = 32
+        self.valid_labels = ["O", "F", "C", "Q"]
+        self.label_to_punct = {"F": "۔", "C": "،", "Q": "؟", "O": ""}
+        self.model = NERModel(
+            "bert",
+            "/code/models/urdu",
+            use_cuda=False,
+            labels=self.valid_labels,
+            args={"silent": True, "max_seq_length": 512},
+        )
+        self.patterns = {
+            "partial": r"[ً-٠ٰ۟-ۤۧ-۪ۨ-ۭ،۔؟]+",
+            "full": string.punctuation + "،؛؟۔٪ء‘’",
+        }
+
+    def punctuation_removal(self, text: str) -> str:
+        if self.normalization == "partial":
+            return re.sub(self.patterns[self.normalization], "", text).strip()
+        else:
+            return "".join(ch for ch in text if ch not in self.patterns[self.normalization])
+
+    def punctuate(self, text: str):
+        text = self.punctuation_removal(text)
+        splits = self.split_on_tokens(text)
+        full_preds_lst = [self.predict(i["text"]) for i in splits]
+        preds_lst = [i[0][0] for i in full_preds_lst]
+        combined_preds = self.combine_results(text, preds_lst)
+        punct_text = self.punctuate_texts(combined_preds)
+        return punct_text
+
+    def predict(self, input_slice):
+        return self.model.predict([input_slice])
+
+    def split_on_tokens(self, text):
+        wrds = text.replace("\n", " ").split()
+        response = []
+        lst_chunk_idx = 0
+        i = 0
+
+        while True:
+            wrds_len = wrds[i * self.wrds_per_pred : (i + 1) * self.wrds_per_pred]
+            wrds_ovlp = wrds[
+                (i + 1) * self.wrds_per_pred : (i + 1) * self.wrds_per_pred + self.overlap_wrds
+            ]
+            wrds_split = wrds_len + wrds_ovlp
+
+            if not wrds_split:
+                break
+
+            response_obj = {
+                "text": " ".join(wrds_split),
+                "start_idx": lst_chunk_idx,
+                "end_idx": lst_chunk_idx + len(" ".join(wrds_len)),
+            }
+
+            response.append(response_obj)
+            lst_chunk_idx += response_obj["end_idx"] + 1
+            i += 1
+
+        return response
+
+    def combine_results(self, full_text: str, text_slices):
+        split_full_text = full_text.replace("\n", " ").split(" ")
+        split_full_text = [i for i in split_full_text if i]
+        split_full_text_len = len(split_full_text)
+        output_text = []
+        index = 0
+
+        if len(text_slices[-1]) <= 3 and len(text_slices) > 1:
+            text_slices = text_slices[:-1]
+
+        for slice in text_slices:
+            slice_wrds = len(slice)
+            for ix, wrd in enumerate(slice):
+                if index == split_full_text_len:
+                    break
+
+                if (
+                    split_full_text[index] == str(list(wrd.keys())[0])
+                    and ix <= slice_wrds - 3
+                    and text_slices[-1] != slice
+                ):
+                    index += 1
+                    pred_item_tuple = list(wrd.items())[0]
+                    output_text.append(pred_item_tuple)
+                elif (
+                    split_full_text[index] == str(list(wrd.keys())[0]) and text_slices[-1] == slice
+                ):
+                    index += 1
+                    pred_item_tuple = list(wrd.items())[0]
+                    output_text.append(pred_item_tuple)
+
+        assert [i[0] for i in output_text] == split_full_text
+        return output_text
+
+    def punctuate_texts(self, full_pred: list):
+        punct_resp = []
+        for punct_wrd, label in full_pred:
+            punct_wrd += self.label_to_punct[label]
+            if punct_wrd.endswith("‘‘"):
+                punct_wrd = punct_wrd[:-2] + self.label_to_punct[label] + "‘‘"
+            punct_resp.append(punct_wrd)
+
+        punct_resp = " ".join(punct_resp)
+        if punct_resp[-1].isalnum():
+            punct_resp += "۔"
+
+        return punct_resp
+
+
+class Urdu:
+    def __init__(self):
+        self.model = BERTmodel()
+
+    def punctuate(self, data: str):
+        return self.model.punctuate(data)