<jupyter_start><jupyter_text>2.3.6 유사단어 랭킹 표시<jupyter_code>def most_similar(query, word_to_id, id_to_word, word_matrix, top=5):
    
    if query not in word_to_id:
        print('%s : no words like that' % query)
        return
    
    print('\n[query]' + query)
    
    query_id = word_to_id[query]
    query_vec = word_matrix[query_id]
    
    vocab_size = len(id_to_word)
    similarity = np.zeros(vocab_size)
    for i in range(vocab_size):
        similarity[i] = cos_similarity(word_matrix[i], query_vec)
        
    count = 0
    for i in (-1 * similarity).argsort():
        if id_to_word[i] == query:
            continue
        print('%s : %s' %(id_to_word[i], similarity[i]))
        
        count += 1
        if count >= top:
            return
from common.util import preprocess, create_co_matrix, most_similar

text = 'you say come over baby I think you are pretty'
corpus, word_to_id, id_to_word = preprocess(text)
vocab_size = len(word_to_id)
C = create_co_matrix(corpus, vocab_size)

most_similar('you', word_to_id, id_to_word, C, top=3)
def most_similar(query, word_to_id, id_to_word, word_matrix, top=5):
    
    if query not in word_to_id:
        print('%s not in the corpus' % query)
        return
    
    print('\n[query]' + query)
    query_id = word_to_id[query]
    query_vec = word_matrix[query_id]
    
    vocab_size = len(id_to_word)
    similarity = np.zeros(vocab_size)
    for i in range(vocab_size):
        similarity[i] = cos_similarity(word_matrix[i], query_vec)
        
    count = 0
    for i in (-1 * similarity).argsort():
        if id_to_word[i] == query:
            continue
        print('%s: %s' % (id_to_word[i], similarity[i]))
        
        count += 1
        if count >= top:
            return
import sys
sys.path.append('..')
from common.util import preprocess, create_co_matrix, most_similar

text = 'you say goodbye and I say hello.'
corpus, word_to_id, id_to_word = preprocess(text)
vocab_size = len(word_to_id)

C = create_co_matrix(corpus, vocab_size)
most_similar('you', word_to_id, id_to_word, C, top = 5)
import numpy as np

def ppmi(C, verbose=False, eps=1e-8):
    
    M = np.zeros_like(C, dtype=np.float32)
    N = np.sum(C)
    S = np.sum(C, axis=0)
    total = C.shape[0]*C.shape[1]
    cnt = 0
    
    for i in range(C.shape[0]):
        for j in range(C.shape[1]):
            pmi = np.log2(C[i, j] * N / (S[j]*S[i]) + eps)
            M[i,j] = max(0, pmi)
            
            if verbose:
                cnt += 1
                if cnt % (total//100) == 0:
                    print('%.1f%% complete' % (100*cnt/total))
                    
    return M

import sys
sys.path.append("..")
import numpy as np
from common.util import preprocess, create_co_matrix, cos_similarity

text = 'you say goodbye and i say hello'
corpus, word_to_id, id_to_word = preprocess(text)
vocab_size = len(word_to_id)
C = create_co_matrix(corpus, vocab_size)
W = ppmi(C)

np.set_printoptions(precision=3)
print(C)
print('-'*50)
print(W)<jupyter_output>[[0 1 0 0 0 0]
 [1 0 1 0 1 1]
 [0 1 0 1 0 0]
 [0 0 1 0 1 0]
 [0 1 0 1 0 0]
 [0 1 0 0 0 0]]
--------------------------------------------------
[[0.    1.585 0.    0.    0.    0.   ]
 [1.585 0.    0.585 0.    0.585 1.585]
 [0.    0.585 0.    1.585 0.    0.   ]
 [0.    0.    1.585 0.    1.585 0.   ]
 [0.    0.585 0.    1.585 0.    0.   ]
 [0.    1.585 0.    0.    0.    0.   ]]<jupyter_text>SVDX = USV^T<jupyter_code>import sys

sys.path.append('..')
import numpy as np
import matplotlib.pyplot as plt
from common.util import preprocess, create_co_matrix, ppmi

text = 'you say goodbye and i say hello'
corpus, word_to_id, id_to_word = preprocess(text)
vocab_size = len(id_to_word)
C= create_co_matrix(corpus, vocab_size, window_size=1)
W = ppmi(C)

U, S, V = np.linalg.svd(W)
print(C[0])
print(W[0])
print(W)
print(U[0])
print(U)
print(S)
print(V)
print(word_to_id)
print([item for item in word_to_id.items()])
for item in word_to_id.items():
    print(item)
for word, word_id in word_to_id.items():
    plt.annotate(word,(U[word_id, 0], U[word_id, 1]))
    
    plt.scatter(U[:,0], U[:,1], alpha=0.5)
    plt.show
import sys
from dataset import ptb

corpus, word_to_id, id_to_word = ptb.load_data('train')

print(len(corpus))
print('corpus[:30]:', corpus[:30])
print(id_to_word[1])
print(word_to_id['hyundai'])
import numpy as np
from common.util import most_similar, create_co_matrix, ppmi
from dataset import ptb

windowsize = 2
wordvec_size = 100

corpus, word_to_id, id_to_word = ptb.load_data('train')
vocab_size = len(word_to_id)
print("common happens")
C = create_co_matrix(corpus, vocab_size, windowsize)
print('ppmi 계산')
W = ppmi(C, verbose = True)

print("SVD")
try:
    from sklearn.utils.extmath import randomized_svd
    U,S,V = randomized_svd(W, n_components=wordvec_size, n_iter=5, random_state=None)
    
except ImportError:
    U,S,V = np.linalg.svd(W)
    
word_vecs = U[:, :wordvec_size]
querys = ['you', 'year', 'car', 'hyundai']

for query in querys:
    most_similar(query, word_to_id, id_to_word, word_vecs, top=5)
import numpy as np

c = np.array([[1,0,0,0,0,0,0]])
W = np.random.randn(7,3)
h = np.matmul(c, W)
print(h)
import sys
sys.path.append('..')
import numpy as np
from common.layers import MatMul

c = np.array([[1,0,0,0,0,0,0]])
W = np.random.randn(7,3)
layer = MatMul(W)
h = layer.forward(c)
print(h)
import sys
sys.path.append('..')
import numpy as np
from common.layers import MatMul

c0 = np.array([[1,0,0,0,0,0,0]])
c1 = np.array([[0,0,1,0,0,0,0]])

W_in = np.random.randn(7,3)
W_out = np.random.randn(3,7)

in_layer0 = MatMul(W_in)
in_layer1 = MatMul(W_in)
out_layer = MatMul(W_out)

h0 = in_layer0.forward(c0)
h1 = in_layer1.forward(c1)
h = 0.5 * (h0 + h1)
s = out_layer.forward(h)

print(s)
def create_contexts_target(corpus, window_size=1):
    target = corpus[window_size:-window_size]
    contexts = []
    
    for idx in range(window_size, len(corpus) - window_size):
        cs = []
        for t in range(-window_size, window_size+1):
            if t == 0:
                continue
            cs.append(corpus[idx+t])
        contexts.append(cs)
        
    return np.array(contexts), np.array(target)
import sys
from common.util import preprocess

text = 'you say goodbye and i say that is none of any sense'
corpus, word_to_id, id_to_word = preprocess(text)
print(corpus)
contexts, target = create_contexts_target(corpus, window_size=1)

print(target, contexts)
import sys
from common.util import preprocess, create_contexts_target, convert_one_hot

text = 'you say goodbye and i say hello'
corpus, word_to_id, id_to_word = preprocess(text)

contexts, target = create_contexts_target(corpus, window_size=1)

print(target)
print(contexts)

vocab_size = len(word_to_id)
target = convert_one_hot(target, vocab_size)
contexts = convert_one_hot(contexts, vocab_size)

print(target)
print(contexts)
import sys
sys.path.append('..')
import numpy as np
from common.layers import MatMul, SoftmaxWithLoss

class SimpleCBOW:
    def __init__(self, vocab_size, hidden_size):
        
        V, H = vocab_size, hidden_size
        W_in = 0.01*np.random.randn(V,H).astype('f')
        W_out = 0.01*np.random.randn(H,V).astype('f')
        
        self.in_layer0 = MatMul(W_in)
        self.in_layer1 = MatMul(W_in)
        self.out_layer = MatMul(W_out)
        self.loss_layer = SoftmaxWithLoss()
        
        layers = [self.in_layer0, self.in_layer1, self.out_layer]
        self.params, self.grads = [], []
        
        for layer in layers:
            self.params += layer.params
            self.grads += layer.grads
        
        self.word_vecs = W_in
def forward(self, contexts, target):
    h0 = self.in_layer0.forward(contexts[:,0])
    h1 = self.in_layer1.forward(contexts[:,1])
    h = (h0+h1) * 0.5
    score = self.out_layer.forward(h)
    loss = self.loss_layer.forward(score, target)
    return loss
def backward(self, dout = 1):
    ds = self.loss_layer.backward(dout)
    da = self.out_layer.backward(ds)
    da *= 0.5
    self.in_layer1.backward(da)
    self.out_layer0.backward(da)
    return None
import sys
sys.path.append('ch03')
from common.trainer import Trainer
from common.optimizer import Adam
from simple_cbow import SimpleCBOW
from common.util import preprocess, create_contexts_target, convert_one_hot

window_size = 1
hidden_size = 5
batch_size = 3
max_epoch = 1000

text = 'if you summon me and tell me I should die, then I will darling. just do not leave me without you.'
corpus, word_to_id, id_to_word = preprocess(text)

vocab_size = len(word_to_id)
contexts, target = create_contexts_target(corpus, window_size)
target = convert_one_hot(target, vocab_size)
contexts = convert_one_hot(contexts, vocab_size)

model = SimpleCBOW(vocab_size, hidden_size)
optimizer = Adam()
trainer = Trainer(model, optimizer)

trainer.fit(contexts, target, max_epoch, batch_size) # 여기서부터 까보기
trainer.plot()
word_vecs = model.word_vecs
for word_id, word in id_to_word.items():
    print(word, word_vecs[word_id])
import sys
sys.path.append('ch03')
import numpy as np
from common.layers import MatMul, SoftmaxWithLoss

class SimpleSkipGram:
    def __init__(self, vocab_size, hidden_size):
        V, H = vocab_size, hidden_size
        
        W_in = 0.01 * random.randn(V,H).astype('f')
        W_out = 0.01 * random.randn(H,V).astype('f')
        
        self.in_layer = MatMul(W_in)
        self.out_layer = MatMul(W_out)
        self.loss_layer1 = SoftmaxWithLoss()
        self.loss_layer2 = SoftmaxWithLoss()
        
        layers = [self.in_layer, self.out_layer]
        self.params, self.grads = [],[]
        for layer in layers:
            self.params += layer.params
            self.grads += layer.grads
            
        self.word_vecs = W_in
        
    def forward(self, contexts, target):
        h = self.in_layer.forward(target)
        s = self.out_layer.forward(h)
        l1 = self.loss_layer1.forward(s, contexts=[:,0])
        l2 = self.loss_layer2.forward(s, contexts=[:,1])
        loss = l1+l2
        return loss<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13169132982421294, 'fraction_numerical': 0.027112920846161487, 'mean_word_length': 3.0376904570970327, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '392741', 'license': 'MIT', 'n_tokens_mistral': 4044, 'n_tokens_neox': 3904, 'n_words': 894, 'path': 'nlpPractice2.ipynb', 'repo_name': 'Sohyun1798/nlpPractice', 'size': '10069'}