<jupyter_start><jupyter_text>Google_translate.py<jupyter_code>#Google Translate
#Built with Google Translation tools
#Copyright 2019 <NAME> MIT License. See LICENSE.
#Jupyter Notebook created by <NAME>

from googleapiclient.discovery import build
import html

def g_translate(source,targetl):
    service = build('translate', 'v2',developerKey=API_KEY)
    request = service.translations().list(q=source, target=targetl)
    response = request.execute()
    return response['translations'][0]['translatedText']

source='Google Translate is great!'
targetl="fr"  
result = g_translate(source,targetl)
print("result:", html.unescape(result))<jupyter_output>result: Google Translate est génial!<jupyter_text>Google_translate_a_few_test_expressions.py<jupyter_code>#Google Translate
#Built with Google Translation tools
#Copyright 2019 <NAME> MIT License. See LICENSE.

from googleapiclient.discovery import build
import html

def g_translate(source,targetl):
    service = build('translate', 'v2',developerKey=API_KEY)
    request = service.translations().list(q=source, target=targetl)
    response = request.execute()
    return response['translations'][0]['translatedText']

source='Hello. My name is Usty!'
targetl="fr"  
result = g_translate(source,targetl)
print("result:",html.unescape(result))


source='The weather is nice today'
targetl="fr"  
result = g_translate(source,targetl)
print("result:",html.unescape(result))


source='Ce professor me cherche des poux.'
targetl="en"  
result = g_translate(source,targetl)
print("result:",html.unescape(result))

source='chercher des poux'
targetl="en"  
result = g_translate(source,targetl)
print("result:",html.unescape(result))
  
source='Chercher des limites est intéressant.'
targetl="en"  
result = g_translate(source,targetl)
print("result:",html.unescape(result))

source='Une SAS ne dispense pas de suivre les recommandations en vigueur autour des pratiques commerciales.'
targetl="en"  
result = g_translate(source,targetl)
print("result:",html.unescape(result))

source='The project team is all ears.'
targetl="fr"  
result = g_translate(source,targetl)
print("result:",html.unescape(result))

source='The coach stopped and everybody was complaning.'
targetl="fr"  
result = g_translate(source,targetl)
print("result:",html.unescape(result))

source='The coach broke down and stopped and everybody was complaning.'
targetl="fr"  
result = g_translate(source,targetl)
print("result:",html.unescape(result))<jupyter_output>result: Bonjour. Je m'appelle Usty!
result: Le temps est beau aujourd'hui
result: This professor is looking for lice.
result: look for lice
result: Finding limits is interesting.
result: A SAS does not exempt from following the recommendations in force around commercial practices.
result: L'équipe du projet est à l'écoute.
result: L'entraîneur s'est arrêté et tout le monde s'est plaint.
result: L'entraîneur est tombé en panne et s'est arrêté et tout le monde se plaignait.<jupyter_text>V1.csv<jupyter_code>%%writefile V1.csv
broke,road,stopped,shouted,class
1,3.5,6.4,9,trainer
1,3.0,5.4,9,trainer
1,3.2,6.3,9,trainer
1,3.1,.5,9,trainer
1,2,8,9,trainer
2,2,8,9,trainer
2,2,8,9,trainer
1,2,8,9,trainer
2,2,8,9,trainer
2,3,8,9,trainer
2,3,8,9,trainer
1,3,8,9,trainer
1,3,8,9,trainer
2,3,8,9,trainer
1,4,8,9,trainer
3,4,8,9,trainer
1,3,8,9,trainer
2,3,8,9,trainer
1,3,8,9,trainer
2,3,8,9.1,trainer
3,3,8,9.3,trainer
1,3,8,9.7,trainer
2,3,8,9.6,trainer
3,3,8,9,trainer
1,3,8,9,trainer
2,3,8,9,trainer
1,3,8,9,trainer
2,3,8,9,trainer
1,2,8,9,trainer
2,3,8,9,trainer
1,3,8,9,trainer
3,3,8,9,trainer
1,4,8,9,trainer
1,5,8,9,trainer
2,1.1,8,9,trainer
1,1.2,8,9,trainer
3,2.5,8,9,trainer
1,1.1,8,9,trainer
3,3.0,7,7,trainer
2,3.4,7,8,trainer
1,3.5,7,9,trainer
2,2.3,7,7,trainer
1,3.2,6,6,trainer
2,3.5,7,9,trainer
3,3.8,8,7,trainer
1,3.0,6,7,trainer
2,3.8,7,7,trainer
2,3.2,8,7,trainer
1,3.7,9,7,trainer
1,3.3,7,7,trainer
2,3.2,9,1,bus
6.4,6.2,9.5,1.5,bus
6.7,7.1,8.7,1.5,bus
5.5,5.3,9.0,1.3,bus
6.5,6.8,9.6,1.5,bus
5.7,7.8,9.5,1.3,bus
6.3,6.3,8.7,1.6,bus
4.7,5.4,8.3,1.0,bus
6.6,6.7,8.6,1.3,bus
5.2,7.7,8.7,1.4,bus
5.0,8.0,8.5,1.0,bus
5.7,9.0,8.2,1.5,bus
6.0,6.2,8.0,1.0,bus
6.1,7.7,7.7,1.4,bus
5.6,5.7,8.6,1.3,bus
6.7,6.1,7.4,1.4,bus
5.6,7.0,8.5,1.5,bus
5.8,8.7,7.1,1.0,bus
6.2,6.2,8.5,1.5,bus
5.6,5.5,6.7,1.1,bus
5.7,6.2,7.8,1.8,bus
6.1,7.8,8.0,1.3,bus
6.3,6.5,9.7,1.5,bus
6.1,7.8,8.7,1.2,bus
6.4,6.7,7.3,1.3,bus
6.6,7.0,7.4,1.4,bus
6.8,8.8,8.8,1.4,bus
6.7,6.0,7.0,1.7,bus
6.0,7.7,8.5,1.5,bus
5.7,8.6,9.5,1.0,bus
5.5,6.4,8.8,1.1,bus
5.5,7.4,9.7,1.0,bus
5.8,8.7,9.7,1.2,bus
6.0,9.7,9.6,1.6,bus
5.4,7.0,9.6,1.5,bus
6.0,6.4,7.5,1.6,bus
6.7,7.1,9.7,1.5,bus
6.3,8.3,9.4,1.3,bus
5.6,7.0,9.1,1.3,bus
5.5,8.5,8.4,1.3,bus
5.5,7.6,9.8,1.2,bus
6.1,6.0,7.6,1.4,bus
5.8,7.6,8.0,1.2,bus
5.0,8.3,9.3,1.0,bus
5.6,9.7,8.2,1.3,bus
5.7,6.0,9.2,1.2,bus
5.7,7.7,9.2,1.3,bus
6.2,6.7,8.3,1.3,bus
5.1,7.5,7.0,1.1,bus
5.7,8.8,8.1,1.3,bus
3.3,7.3,3.0,2.5,coach
4.7,5.7,3.1,3.7,coach
2.0,6.0,2.7,3.1,coach
2.7,5.7,3.6,1.8,coach
3.0,4.0,2.8,2.2,coach
3.0,5.0,3.6,2.1,coach
2.5,4.5,2.5,1.7,coach
2.7,5.7,3.3,1.8,coach
2.5,4.5,3.8,1.8,coach
3.6,5.6,2.1,2.5,coach
3.2,6.2,4.1,2.0,coach
2.7,5.7,3.3,1.7,coach
3.0,6.0,4.5,2.1,coach
2.5,5.5,3.0,2.0,coach
2.8,4.8,4.1,2.4,coach
3.2,5.2,3.3,2.3,coach
3.0,6.0,4.5,1.8,coach
3.8,4.8,3.7,2.2,coach
2.6,5.6,4.7,2.3,coach
2.2,2.2,3.0,1.5,coach
3.2,3.2,4.7,2.3,coach
2.8,2.8,3.7,2.0,coach
2.8,2.8,3.7,2.0,coach
2.7,2.7,1.7,1.8,coach
3.3,3.3,3.7,2.1,coach
3.2,3.2,3.0,1.8,coach
2.8,2.8,3.8,1.8,coach
3.0,3.0,4.7,1.8,coach
2.8,2.8,3.6,2.1,coach
3.0,3.0,4.8,1.6,coach
2.8,2.8,3.1,1.7,coach
3.8,3.8,2.4,2.0,coach
2.8,2.8,3.6,2.2,coach
2.8,2.8,1.1,1.5,coach
2.6,2.6,2.6,1.4,coach
3.0,3.0,4.1,2.3,coach
3.4,3.4,2.6,2.4,coach
3.1,3.1,3.5,1.8,coach
3.0,3.0,2.8,1.8,coach
3.1,3.1,2.4,2.1,coach
3.1,3.1,4.6,2.4,coach
3.1,3.1,3.1,2.3,coach
2.7,2.7,4.1,1.7,coach
3.2,3.2,2.7,2.3,coach
3.3,3.3,3.7,2.5,coach
3.0,3.0,4.2,2.3,coach
2.5,2.5,2.0,1.7,coach
3.0,3.0,3.2,2.0,coach
3.4,3.4,2.4,2.3,coach
3.0,3.0,3.1,1.8,coach<jupyter_output>Writing V1.csv<jupyter_text>Google_Translate_Customized.py<jupyter_code>#Google Translate
#Built with Google Translation tools
#Copyright 2019 <NAME> MIT License. See LICENSE.
from googleapiclient.discovery import build
import pandas as pd
from sklearn.neighbors import KNeighborsClassifier

X=['Eating fatty food can be unhealthy.',
   'This was a catch-22 situation.',
   'She would not lend me her tote bag',
   'He had a chip on his shoulder',
   'The market was bearish yesterday',
   'That was definitely wrong',
   'The project was compromised but he pulled a rabit out of his hat',
   'So just let the chips fall where they may',
   'She went the extra mile to satisfy the customer',
   'She bailed out when it became unbearable',
   'The term person includes one or more individuals, labor unions, partnerships, associations, corporations, legal representatives, mutual companies, joint-stock companies, trusts, unincorporated organizations, trustees, trustees in bankruptcy, or receivers.',
   'The coach broke down, stopped and everybody was complaining']

X1=['grasse',
    'insoluble',
    'sac',
    'aggressif',
    'marché',
    'certainement',
    'chapeau',
    'advienne',
    'supplémentaire',
    'parti',
    'personne',
    'bus']


X2=[0,0,0,1,0,0,0,0,0,0,0,1]

phrase_translation=['','','','Il est agressif','','','','','','','','']
V1=['broke','road','stopped','shouted','coach','bus','car','truck','break','broke','roads','stop']
V1_class=['trainer','coach','bus']
vpolysemy=[[0,0,0,0]]

def g_translate(source,targetl,m):
    service = build('translate', 'v2',developerKey=API_KEY)
    request = service.translations().list(q=source, target=targetl,format='text',model=m)
    response = request.execute()
    return response['translations'][0]['translatedText']

def deeper_translate(source,index):
    dt=source
    deeper_response=phrase_translation[index]
    if(len(deeper_response)<=0):
        print("deeper translation program result:",deeper_response,":Now true")
    if(len(deeper_response)<=0):
        v1=0
        for i in range(4):
            ngram=V1[i]
            if(ngram in source):
                vpolysemy[0][i]=9
                v1=1
        if(v1>0):
            polysemy='V1'
            begin=str(V1[0]).strip('[]');end=str(V1[3]).strip('[]')
            sememe=knn(polysemy,vpolysemy,begin,end)
            for i in range(2):
                if(V1_class[i] in source):
                    replace=str(V1_class[i]).strip('[]')
                    sememe=str(sememe).strip('[]')
                    dtsource = source.replace(replace,sememe)
                    targetl="fr";m='base'  
                    result = g_translate(dtsource,targetl,m)
                    print('polysemy narrowed result:',result,":Now true")
    
                    
def knn(polysemy,vpolysemy,begin,end):
    df = pd.read_csv(polysemy+'.csv')
    X = df.loc[:,begin:end]
    Y = df.loc[:,'class']
    knn = KNeighborsClassifier()
    knn.fit(X,Y)
    prediction = knn.predict(vpolysemy)
    return prediction

def frequency_p(tnumber,cnumber):
    ff=cnumber/tnumber  #frequentist interpretation and probablity
    return ff

#print('Phrase-Based Machine Translation(PBMT)model:base'): #m='base'  

print('Neural Machine Translation model:nmt')
t=0;f=0;dt=0     # true;false;deeper translation call
for xi in range(len(X)):
    source=X[xi]
    targetl="fr";m='nmt'  
    result = g_translate(source,targetl,m)
    targetl="en"
    back_translate=result
    back_translate = g_translate(back_translate,targetl,m)
    print("source:",source,":",len(source))
    print("result:",result)
    print("target:",back_translate,":",len(back_translate))
    #length comparaison can be used to improve the algorithm
    #In this case, the source is compared to the back translation
    term=X1[xi]
    print("term:",term)
    input=result
    words=input.split()
    if(source == back_translate):
          print("true")
          if((term not in words)and (xi!=4)):
              t+=1
    else:
        f+=1;print("false")
        if(X2[xi]>0):    
            DT=deeper_translate(source,xi)
            dt+=1
    if(f>0):
        B1=frequency_p(xi+1,f)    #error detection probablity before deep translation 
        B2=frequency_p(xi+1,f-dt) #error detection probablity after deep translation
    if(f>0):
        print("ETP before DT",round(B1,2),"ETP with DT",round(B2,2))
    else:
        print('Insufficient data in probablity distribution')
      
print("------Summary------")
print('Neural Machine Translation model:nmt')
print('Google Translate:',"True:",t,"False:",f,'ETP',round(f/len(X),2))
print('Customized Google Translate:',"True:",t,"False:",f-dt,'ETP',round((f-dt)/len(X),2))
a=2.5;at=t+a;af=f-a #subjective acceptance of an approximate result
print('Google Translate acceptable:',"True:",at,"False:",af,'ETP',round(af/len(X),2))
#The error rate should decrease and be stablized as the KNN knowledge base increases
print('Customized Google Translate acceptable:',"True:",at,"False:",af-dt,'ETP',round((af-dt)/len(X),2))<jupyter_output>Neural Machine Translation model:nmt
source: Eating fatty food can be unhealthy. : 35
result: Manger des aliments gras peut être malsain.
target: Eating fatty foods can be unhealthy. : 36
term: grasse
false
ETP before DT 1.0 ETP with DT 1.0
source: This was a catch-22 situation. : 30
result: Ce fut une situation de catch-22.
target: It was a catch-22 situation. : 28
term: insoluble
false
ETP before DT 1.0 ETP with DT 1.0
source: She would not lend me her tote bag : 34
result: Elle ne me prêterait pas son sac fourre-tout
target: She wouldn't lend me her tote bag : 33
term: sac
false
ETP before DT 1.0 ETP with DT 1.0
source: He had a chip on his shoulder : 29
result: Il avait une puce sur l'épaule
target: He had a chip on his shoulder : 29
term: aggressif
true
ETP before DT 0.75 ETP with DT 0.75
source: The market was bearish yesterday : 32
result: Le marché était baissier hier
target: Market was bearish yesterday : 28
term: marché
false
ETP before DT 0.8 ETP with DT 0.8
source: That was[...]<jupyter_text>knn_polysemy.py<jupyter_code>#K-means clustering 
#Build with Sklearn
#Copyright 2019 <NAME> MIT License. See LICENSE.
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.neighbors import KNeighborsClassifier

# Importthe persed data
df = pd.read_csv('V1.csv')

print (df.head())

# KNN Classification labels
X = df.loc[:,'broke':'shouted']
Y = df.loc[:,'class']

# Trains the model
knn = KNeighborsClassifier()
knn.fit(X,Y)

# Requesting a prediction
#broke and stopped are
#activated to see the best choice of words to fit these features.
# brock and stopped were found in the sentence to be interpreted.
# In X_DL as in X, the labels are : broke, road, stopped,shouted.
X_DL = [[9,0,9,0]] 
prediction = knn.predict(X_DL)
print ("The prediction is:",str(prediction).strip('[]'))

#Uses the same V1.txt because the parsing has
# been checked and is reliable as "dataset lexical rule base".
df = pd.read_csv('V1.csv') 
# Plotting the relation of each feature with each class
figure,(sub1,sub2,sub3,sub4)=plt.subplots(4,sharex=True,sharey=True)
plt.suptitle('k-nearest neighbors')
plt.xlabel('Feature')
plt.ylabel('Class') 
X = df.loc[:,'broke']
Y = df.loc[:,'class']
sub1.scatter(X, Y,color='blue',label='broke')
sub1.legend(loc=4, prop={'size': 5})
sub1.set_title('Polysemy')
X = df.loc[:,'road']
Y = df.loc[:,'class']
sub2.scatter(X, Y,color='green',label='road')
sub2.legend(loc=4, prop={'size': 5})
X = df.loc[:,'stopped']
Y = df.loc[:,'class']
sub3.scatter(X, Y,color='red',label='stopped')
sub3.legend(loc=4, prop={'size': 5})
X = df.loc[:,'shouted']
Y = df.loc[:,'class']
sub4.scatter(X, Y,color='black',label='shouted')
sub4.legend(loc=4, prop={'size': 5})
figure.subplots_adjust(hspace=0)
plt.show()<jupyter_output>broke  road  stopped  shouted    class
0    1.0   3.5      6.4      9.0  trainer
1    1.0   3.0      5.4      9.0  trainer
2    1.0   3.2      6.3      9.0  trainer
3    1.0   3.1      0.5      9.0  trainer
4    1.0   2.0      8.0      9.0  trainer
The prediction is: 'bus'
{'fraction_non_alphanumeric': 0.18045325779036828, 'fraction_numerical': 0.08781869688385269, 'mean_word_length': 4.328679245283019, 'pattern_counts': {'":': 4, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '224542', 'license': 'MIT', 'n_tokens_mistral': 6301, 'n_tokens_neox': 6171, 'n_words': 1152, 'path': 'CH06/COLAB_Translate.ipynb', 'repo_name': 'PacktPublishing/Artificial-Intelligence-By-Example-Second-Edition', 'size': '14155'}