<jupyter_start><jupyter_text>Rescaling===再スケーリング色々<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
FIG_DIR = 'img_rescaling/'<jupyter_output><empty_output><jupyter_text>もとの分布<jupyter_code>q = np.array([0.1,0.05,0.6,0,0.2,0.05])
I = np.arange(q.shape[0])
plt.bar(I,q)
plt.title('Original distribution : q')
plt.ylim([0,1])
plt.ylabel('Probability')
plt.savefig(FIG_DIR + 'q.png')<jupyter_output><empty_output><jupyter_text>一様分布<jupyter_code>u = np.ones(q.shape[0]) / q.shape[0]

plt.bar(I,u)
plt.title('Uniform distribution : u')
plt.ylim([0,1])
plt.ylabel('Probability')
plt.savefig(FIG_DIR + 'uniform.png')<jupyter_output><empty_output><jupyter_text>One-Hot分布<jupyter_code>v = np.zeros(q.shape[0])
v[q.argmax()] = 1

plt.bar(I,v)
plt.title('One-hot distribution : v')
plt.ylim([0,1])
plt.ylabel('Probability')
plt.savefig(FIG_DIR + 'one_hot.png')<jupyter_output><empty_output><jupyter_text>先鋭化と平滑化<jupyter_code>plt.figure(figsize=(15,3))

# smoothing
plt.subplot(1,3,1)
T = 0.2
p = np.exp(q/T) / np.sum(np.exp(q/T))
plt.bar(I,p)
plt.title('Sharpened probability')
plt.ylim([0,1])
plt.xticks(np.arange(6))
plt.ylabel('Probability')

# original
plt.subplot(1,3,2)
plt.bar(I,q,color = 'orange')
plt.title('Original distribution : q')
plt.xticks(np.arange(6))
plt.ylim([0,1])


# sharpening
plt.subplot(1,3,3)
T = 0.5
p = np.exp(q/T) / np.sum(np.exp(q/T))
plt.bar(I,p)
plt.title('Smoothed probability')
plt.xticks(np.arange(6))
plt.ylim([0,1])
plt.savefig(FIG_DIR + 'sample.png')<jupyter_output><empty_output><jupyter_text>Thermal Rescaling<jupyter_code>Ts = 10**np.arange(0,1,0.001)+0.05 - 1

Du = np.zeros(Ts.shape[0])
Dv = np.zeros(Ts.shape[0])
Dq = np.zeros(Ts.shape[0])

for i in range(Ts.shape[0]):
    T = Ts[i]
    p = np.exp(q/T) / np.sum(np.exp(q/T))
    Du[i] = np.sum(u[u>0]*np.log(u[u>0]) - u[u>0] * np.log(p[u>0]))
    Dv[i] = np.sum(v[v>0]*np.log(v[v>0]) - v[v>0] * np.log(p[v>0]))
    Dq[i] = np.sum(q[q>0]*np.log(q[q>0]) - q[q>0] * np.log(p[q>0]))



plt.plot(Ts,Dq,label='Distance to original distribution :D(q||p)')
plt.plot(Ts,Du,label='Distance to uniform distribution : D(u||p)')
plt.plot(Ts,Dv,label='Distance to one-hot distribution :D(v||p)')
plt.yscale('log')
plt.xscale('log')
plt.legend()

plt.xlabel('Thermal : T')
plt.ylabel('Distance')

plt.title('KL divergence of Thermal Rescaling')
plt.savefig(FIG_DIR + 'KL_thermal.png')
plt.figure(figsize=(15,6))
plt.subplots_adjust(hspace=0.3)

# original
plt.subplot(2,3,1)
plt.bar(I,q,color = 'orange')
plt.title('Original distribution : q')
plt.xticks(np.arange(6))
plt.ylim([0,1])
plt.ylabel('Probability')


# T=0.05
plt.subplot(2,3,2)
T = 0.05
p = np.exp(q/T) / np.sum(np.exp(q/T))
plt.bar(I,p)
plt.title('T = {0:.02f}'.format(T))
plt.ylim([0,1])
plt.xticks(np.arange(6))

# T=0.2
plt.subplot(2,3,3)
T = 0.15
p = np.exp(q/T) / np.sum(np.exp(q/T))
plt.bar(I,p)
plt.title('T = {0:.02f}'.format(T))
plt.ylim([0,1])
plt.xticks(np.arange(6))

# T=optimal
plt.subplot(2,3,4)
T = Ts[np.argmin(Dq)]
p = np.exp(q/T) / np.sum(np.exp(q/T))
plt.bar(I,p)
plt.title('Minimize D(q||p(T)) : T = {0:.02f}'.format(T))
plt.ylim([0,1])
plt.xticks(np.arange(6))
plt.ylabel('Probability')


plt.subplot(2,3,5)
T = 1
p = np.exp(q/T) / np.sum(np.exp(q/T))
plt.bar(I,p)
plt.title('T = {0:.02f}'.format(T))
plt.ylim([0,1])
plt.xticks(np.arange(6))

plt.subplot(2,3,6)
T = 10
p = np.exp(q/T) / np.sum(np.exp(q/T))
plt.bar(I,p)
plt.title('T = {0:.02f}'.format(T))
plt.ylim([0,1])
plt.xticks(np.arange(6))

plt.savefig(FIG_DIR + 'example_thermal.png')<jupyter_output><empty_output><jupyter_text>Frozen Rescaling<jupyter_code>Ts = 10**np.arange(0,1,0.001)+0.2 - 1


Du = np.zeros(Ts.shape[0])
Dv = np.zeros(Ts.shape[0])
Dq = np.zeros(Ts.shape[0])

for i in range(Ts.shape[0]):
    T = Ts[i]
    p = q**(1/T) / np.sum(q**(1/T))
    Dq[i] = np.sum(q[q>0]*np.log(q[q>0]) - q[q>0] * np.log(p[q>0]))
    Du[i] = np.sum(p[p>0]*np.log(p[p>0]) - p[p>0] * np.log(u[p>0]))
    Dv[i] = np.sum(v[v>0]*np.log(v[v>0]) - v[v>0] * np.log(p[v>0]))



plt.plot(Ts,Dq,label='Distance to original distribution :D(q||p)')
plt.plot(Ts,Du,label='Distance to uniform distribution : D(p||u)')
plt.plot(Ts,Dv,label='Distance to one-hot distribution :D(v||p)')
plt.yscale('log')
plt.xscale('log')
plt.legend()

plt.xlabel('Thermal : T')
plt.ylabel('Distance')

plt.title('KL divergence of Frozen Function')
plt.savefig(FIG_DIR + 'KL_freesing.png')
plt.figure(figsize=(15,6))
plt.subplots_adjust(hspace=0.3)

# original
plt.subplot(2,3,1)
plt.bar(I,q,color = 'orange')
plt.title('Original distribution : q')
plt.xticks(np.arange(6))
plt.ylim([0,1])
plt.ylabel('Probability')


plt.subplot(2,3,2)
T = 0.2
p = q**(1/T) / np.sum(q**(1/T))
plt.bar(I,p)
plt.title('T = {0:.02f}'.format(T))
plt.ylim([0,1])
plt.xticks(np.arange(6))

plt.subplot(2,3,3)
T = 0.5
p = q**(1/T) / np.sum(q**(1/T))
plt.bar(I,p)
plt.title('T = {0:.02f}'.format(T))
plt.ylim([0,1])
plt.xticks(np.arange(6))

# T=optimal
plt.subplot(2,3,4)
T = Ts[np.argmin(Dq)]
p = q**(1/T) / np.sum(q**(1/T))
plt.bar(I,p)
plt.title('Minimize D(q||p(T)) : T = {0:.02f}'.format(T))
plt.ylim([0,1])
plt.xticks(np.arange(6))
plt.ylabel('Probability')


plt.subplot(2,3,5)
T = 2
p = q**(1/T) / np.sum(q**(1/T))
plt.bar(I,p)
plt.title('T = {0:.02f}'.format(T))
plt.ylim([0,1])
plt.xticks(np.arange(6))

plt.subplot(2,3,6)
T = 10
p = q**(1/T) / np.sum(q**(1/T))
plt.bar(I,p)
plt.title('T = {0:.02f}'.format(T))
plt.ylim([0,1])
plt.xticks(np.arange(6))

plt.savefig(FIG_DIR + 'example_freesing.png')<jupyter_output><empty_output><jupyter_text>Example by NLP<jupyter_code>corpus = "I have a pen. I have a dog. I buy a pen."
# 文に分ける
sentences =  corpus.split('.')
# 空文の削除と[SoS],[EoS]の付加
sentences = ['[SoS] ' + s + ' [EoS]' for s in sentences if s != '']
# [EoS]→[EoS]の追加
sentences.append("[EoS] [EoS]")
# ダブルスペースの削除
sentences = [s.replace('  ',' ')  for s in sentences]
# ワードに分ける
words =[]
for s in sentences:
    words.append(s.split(' '))
# ワードリストの作成
word_list = ['[SoS]'] + list(set(sum(words,[])) - set(['[SoS]','[EoS]'])) + ['[EoS]']
# ワードリストを要素番号に変換
num_list = np.arange(len(word_list))
word_dict = dict(zip(word_list,num_list))
#遷移確率の作成
A = np.zeros((len(word_list),len(word_list)))
for s in words:
    for i in range(len(s) - 1):
        #数え上げ         
        A[word_dict[s[i+1]],word_dict[s[i]]] +=1
A = A / A.sum(axis = 0)
# Originalの文書生成
sentences_g = []

for i in range(10):
    #最初は[SoS]
    w = [0]
    while True:
        # 次の確率分布
        q = A[:,w[-1]]
        # sampling and append
        w.append(np.argmax(np.random.multinomial(1,q)))
        # if EoS then break
        if w[-1] == len(word_list)-1:
            break;
    # 文字列に変換            
    s = ''
    for i in w:
        s = s + ' ' +word_list[i]  

    #文書の追加 
    sentences_g.append(s)
# 表示
for s in sentences_g:
    print(s)
#  Sharpening by Freesing : T = 0.1
T = 0.1
sentences_g = []
for i in range(10):
    w = [0]
    while True:
        q = A[:,w[-1]]
        # 再スケーリング
        p = q**(1/T) / np.sum(q**(1/T))
        w.append(np.argmax(np.random.multinomial(1,p)))
        if w[-1] == len(word_list)-1:
            break;
    s = ''
    for i in w:
        s = s + ' ' +word_list[i]  
    sentences_g.append(s)
for s in sentences_g:
    print(s)
#  Smoothing by Freesing : T = 2.0
T = 2.0
sentences_g = []
for i in range(10):
    w = [0]
    while True:
        q = A[:,w[-1]]
        # 再スケーリング
        p = q**(1/T) / np.sum(q**(1/T))
        w.append(np.argmax(np.random.multinomial(1,p)))
        if w[-1] == len(word_list)-1:
            break;
    s = ''
    for i in w:
        s = s + ' ' +word_list[i]  
    sentences_g.append(s)
for s in sentences_g:
    print(s)
#  Smoothing by Thermal : T = 1.0
T = 1.0
sentences_g = []
for i in range(10):
    w = [0]
    while True:
        q = A[:,w[-1]]
        # 再スケーリング
        p = np.exp(q/T) / np.sum(np.exp(q/T))
        w.append(np.argmax(np.random.multinomial(1,p)))
        if w[-1] == len(word_list)-1:
            break;
    s = ''
    for i in w:
        s = s + ' ' +word_list[i]  

    sentences_g.append(s)
for s in sentences_g:
    print(s)
N = 100000
u = np.random.rand(N)
plt.hist(u,bins = 100)
plt.show()
pi = np.log(np.random.rand(N))
plt.hist(pi,bins = 100)
plt.show()
g = -np.log(-np.log(u))
plt.hist(g,bins = 100)
plt.show()
plt.hist(g+u,bins = 100)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.23315315315315316, 'fraction_numerical': 0.03783783783783784, 'mean_word_length': 3.80715935334873, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '143516', 'license': 'Apache-2.0', 'n_tokens_mistral': 4393, 'n_tokens_neox': 4105, 'n_words': 603, 'path': 'Rescaling.ipynb', 'repo_name': 'yuji0001/2020GenerativeModel', 'size': '8325'}