<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from time import sleep
from google.cloud import translate
from IPython.display import clear_output

SOURCE_LANG = 'en'


class Translate:
    def __init__(self,target_lang):
        self.translate_client = translate.Client()
        self.target_lang = target_lang
    
    def get_translation(self, text, nmt=True):
        text = text.replace("\n", " ")
        text = text.replace("\\", "")
        #Call translation api with NMT
        model = translate.NMT if nmt else translate.BASE
        result = self.translate_client.translate(
            text
            , source_language=SOURCE_LANG
            , target_language=self.target_lang
            , model=model
        )
        #Return the translated result
        return result['translatedText']
    
    def check_arxiv(self, bulk, nmt=True):
        N = len(bulk)
        lc = 0
        breaker = None
        
        while not breaker:
            title = bulk[lc]['title']
            pdf_url = bulk[lc]['pdf_url']
            abst = bulk[lc]['summary']
            
            translation = self.get_translation(abst, nmt)
            
            print()
            print("{} / {}(Total number)".format(lc+1,N))
            print()
            print(title)
            print()
            print(pdf_url)
            print()
            print(translation)
            
            sleep(0.3)<commit_msg>Change to show the original text<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from time import sleep
from google.cloud import translate
from IPython.display import clear_output

SOURCE_LANG = 'en'


class Translate:
    def __init__(self,target_lang):
        self.translate_client = translate.Client()
        self.target_lang = target_lang
    
    def get_translation(self, text, nmt=True):
        text = text.replace("\n", " ")
        text = text.replace("\\", "")
        #Call translation api with NMT
        model = translate.NMT if nmt else translate.BASE
        result = self.translate_client.translate(
            text
            , source_language=SOURCE_LANG
            , target_language=self.target_lang
            , model=model
        )
        #Return the translated result
        return result['translatedText'], text
    
    def check_arxiv(self, bulk, nmt=True):
        N = len(bulk)
        lc = 0
        breaker = None
        
        while not breaker:
            title = bulk[lc]['title']
            pdf_url = bulk[lc]['pdf_url']
            abst = bulk[lc]['summary']
            
            translation, org = self.get_translation(abst, nmt)
            
            print()
            print("{} / {}(Total number)".format(lc+1,N))
            print()
            print(title)
            print()
            print(pdf_url)
            print()
            print(org)
            print()
            print(translation)
            
            sleep(0.3)
{'fraction_non_alphanumeric': 0.10594227504244483, 'fraction_numerical': 0.003395585738539898, 'mean_word_length': 1.687956204379562, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '312017', 'n_tokens_mistral': 819, 'n_tokens_neox': 801, 'n_words': 188}