<jupyter_start><jupyter_text>Word2Vec trên một từ đơn<jupyter_code>nn_idx = wordsList.index('nhà_nước')
print('Index of `nhà_nước` in wordsList:', nn_idx)
nn_vec = wordVectors[nn_idx]
print('Length of Vector representation of `nhà_nước`:', len(nn_vec))<jupyter_output>Index of `nhà_nước` in wordsList: 10829
Length of Vector representation of `nhà_nước`: 300<jupyter_text>Word2Vec để biểu diễn một đoạn văn bản<jupyter_code>sentence = '<NAME> ăn hoài không biết chán'
maxSeqLength = 10   #Maximum length of sentence
numDimensions = 300 #Dimensions for each word vector
sentenceIndexes = np.zeros((maxSeqLength), dtype='int32')

word_in_sentence = [word.lower() for word in sentence.split()]
for i in range(len(word_in_sentence)):
    idx =  wordsList.index(word_in_sentence[i])
    sentenceIndexes[i] = idx
    
print(sentenceIndexes.shape)
print('Row index for each word: ', sentenceIndexes)

# Ma trận biểu diễn:
print('Sentence representation of word vectors:')
with tf.Session() as sess:
    print(tf.nn.embedding_lookup(wordVectors,sentenceIndexes).eval().shape)
wordsList = np.load('wordsList.npy')
wordsList = wordsList.tolist()
wordVectors = np.load('wordVectors.npy')
wordVectors = np.float32(wordVectors)
nn_idx = wordsList.index('nhà_nước')
print('Index of `nhà_nước` in wordsList:', nn_idx)
nn_vec = wordVectors[nn_idx]
print('Vector representation of `nhà_nước`:', nn_vec)
sentence = '<NAME>ày ăn hoài không biết chán'
maxSeqLength = 10
numDimensions = 300
sentenceIndexes = np.zeros((maxSeqLength), dtype=np.int32)

word_in_sentence = [word.lower() for word in sentence.split()]
for i in range(len(word_in_sentence)):
    idx =  wordsList.index(word_in_sentence[i])
    sentenceIndexes[i] = idx

sentenceIndexes
with tf.Session() as sess:
    lookup_words = sess.run(tf.nn.embedding_lookup(wordVectors, sentenceIndexes))

lookup_words.shape    
positiveFiles = ['positiveReviews/' + f for f in listdir('positiveReviews/') if isfile(join('positiveReviews/', f))]
negativeFiles = ['negativeReviews/' + f for f in listdir('negativeReviews/') if isfile(join('negativeReviews/', f))]
numWords = []
for pf in positiveFiles:
    with open(pf, 'r', encoding='utf-8') as f:
        line = f.readline()
        counter = len(line.split())
        numWords.append(counter)
for nf in negativeFiles:
    with open(nf, 'r', encoding='utf-8') as f:
        line = f.readline()
        counter = len(line.split())
        numWords.append(counter)
numFiles = len(numWords)
print('The total number of files is', numFiles)
print('The total number of words in the files is', sum(numWords))
print('The average number of words in the files is', sum(numWords)/len(numWords))
plt.hist(numWords, 50)
plt.xlabel('Sequence Length')
plt.ylabel('Frequency')
plt.axis([0, 400, 0, 4000])
maxSeqLength = 180
print('A positive sentence:')
fname = positiveFiles[3]
with open(fname, encoding='utf-8') as f:
    line = f.readline()
    print(line)

print('A negative sentence:')
fname = negativeFiles[3]
with open(fname, encoding='utf-8') as f:
    line = f.readline()
    print(line)
import re
strip_special_chars = re.compile('[^\w0-9 ]+')

def cleanSentences(string):
    string = string.lower().replace("<br />", " ")
    return re.sub(strip_special_chars, "", string.lower())
ids = np.zeros((numFiles, maxSeqLength), dtype=np.int32)
nFiles = 0
unk_idx = wordsList.index('UNK')

for pf_idx in range(len(positiveFiles)):
    with open(positiveFiles[pf_idx], 'r', encoding='utf-8') as f:
        # nIndexes = 0
        line = f.readline()
        cleanedLine = cleanSentences(line)
        split = cleanedLine.split()
        for word_idx in range(len(split)):
            if word_idx >= maxSeqLength:
                break
            try:
                idx = wordsList.index(split[word_idx])
                ids[pf_idx, word_idx] = idx
            except ValueError:
                ids[pf_idx, word_idx] = unk_idx

for nf_idx in range(len(positiveFiles), len(positiveFiles)+len(negativeFiles)):
    with open(negativeFiles[nf_idx-len(positiveFiles)], 'r', encoding='utf-8') as f:
        # nIndexes = 0
        line = f.readline()
        cleanedLine = cleanSentences(line)
        split = cleanedLine.split()
        for word_idx in range(len(split)):
            if word_idx >= maxSeqLength:
                break
            try:
                idx = wordsList.index(split[word_idx])
                ids[nf_idx, word_idx] = idx
            except ValueError:
                ids[nf_idx, word_idx] = unk_idx
            
np.save('idsMatrix.npy', ids)
len(positiveFiles)
with open(positiveFiles[0], 'r', encoding='utf-8') as f:
    line = f.readline()
    cleanedLine = cleanSentences(line)
    split = cleanedLine.split()
    print(line)
    print(cleanedLine)
ids = np.load('idsMatrix.npy')
ids[0]
ids[0:1]
def getTrainBatch():
    labels = []
    arr = np.zeros((batchSize, maxSeqLength))
    for i in range(batchSize):
        if i % 2 == 0:
            # Pick positive samples randomly
            num = randint(1, 14000)
            labels.append([1, 0])
        else:
            # Pick negative samples randomly
            num = randint(16001, 30000)
            labels.append([0, 1])
            
        arr[i] = ids[num-1:num]
    return arr, labels

def getTestBatch():
    labels = []
    arr = np.zeros((batchSize, maxSeqLength))
    for i in range(batchSize):
        num = randint(14001, 16000)
        if num <= 15000:
            labels.append([1, 0])
        else:
            labels.append([0, 1])
        arr[i] = ids[num-1:num]
    return arr, labels
# Initialize paramters
numDimensions = 300
batchSize = 64
lstmUnits = 128
nLayers = 2
numClasses = 2
iterations = 30000
tf.reset_default_graph()
labels = tf.placeholder(tf.float32, shape=[batchSize, numClasses], name='labels')
inputs = tf.placeholder(tf.int32, shape=[batchSize, maxSeqLength], name='inputs')
data = tf.Variable(tf.zeros((batchSize, maxSeqLength, numDimensions)), dtype=tf.float32)
data = tf.nn.embedding_lookup(wordVectors, inputs)
def generate_a_lstm_layer():
    basic_lstm = tf.contrib.rnn.BasicLSTMCell(num_units=lstmUnits)
    drop_out = tf.contrib.rnn.DropoutWrapper(cell=basic_lstm, output_keep_prob=0.75)
    return drop_out

multi_layer = tf.nn.rnn_cell.MultiRNNCell([generate_a_lstm_layer() for _ in range(nLayers)])
outputs, states = tf.nn.dynamic_rnn(multi_layer, data, dtype=tf.float32)
x = tf.constant([[[ 1,  2,  3],
                  [ 4,  5,  6]],
                 [[ 7,  8,  9],
                  [10, 11, 12]]])
print(x.shape)
x = tf.transpose(x, perm=[1, 2, 0])
x.shape
x = tf.constant([[[1, 4], [2, 5], [3, 6]], [[7, 10], [8, 11], [9, 12]]])
x.shape
weight = tf.Variable(tf.truncated_normal([lstmUnits, numClasses]))
bias = tf.Variable(tf.constant(0.1, shape=[numClasses]))

outputs = tf.transpose(outputs, [1, 0 , 2])
last = tf.gather(outputs, int(outputs.get_shape()[0]) - 1)

predict = tf.matmul(last, weight) + bias
correct_result = tf.equal(tf.argmax(predict, 1), tf.argmax(labels, 1))
accuracy = tf.reduce_mean(tf.cast(correct_result, dtype=tf.float32))
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predict, labels=labels))
optimizer = tf.train.AdamOptimizer().minimize(loss)
import datetime

tf.summary.scalar('Loss', loss)
tf.summary.scalar('Accuracy', accuracy)
merged = tf.summary.merge_all()
logdir = "tensorboard/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + "/"
writer = tf.summary.FileWriter(logdir, sess.graph)
# sess = tf.InteractiveSession()
# saver = tf.train.Saver()
# sess.run(tf.global_variables_initializer())

# for i in range(iterations):
#     next_batch, next_batch_labels = getTrainBatch()
#     sess.run(optimizer, feed_dict={inputs: next_batch, labels: next_batch_labels})
    
#     if (i % 50 == 0):
#         summary = sess.run(merged, {inputs: next_batch, labels: next_batch_labels})
#         writer.add_summary(summary, i)

#     # Save model every 2000 training iterations
#     if (i % 2000 == 0 and i != 0):
#         save_path = saver.save(sess, "models/pretrained_lstm.ckpt", global_step=i)
#         print("saved to %s" % save_path)
# writer.close()
sess = tf.Session()
sess.run(tf.global_variables_initializer())
sess = tf.InteractiveSession()
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint('models_3'))
input_sentence = 'Món này ăn hơi đắng'
# input_sentence = ViTokenizer.tokenize(input_sentence)
input_sentence = word_tokenize(input_sentence, format='text')
#input_sentence = 'Món này ăn ngon mê ly luôn. Vị ngọt và thơm quá trời quá đất.'
#input_sentence = 'Món bánh_bao này ăn có_vẻ đã để lâu'

def input_to_batch_size(input_sentence):
    arr = np.zeros((batchSize, maxSeqLength), dtype=np.int32)
    cleanedLine = cleanSentences(input_sentence)
    split = cleanedLine.split()
    unk_idx = wordsList.index('UNK')
    
    tmp = np.zeros(maxSeqLength)
    for word_idx in range(len(split)):
        if word_idx >= maxSeqLength:
            break
        try:
            idx = wordsList.index(split[word_idx])
            tmp[word_idx] = idx
        except ValueError:
            tmp[word_idx] = unk_idx
    
    arr[0] = tmp
    return arr

input_sentence = input_to_batch_size(input_sentence)
input_sentence.shape
predict_sentence = sess.run(predict, feed_dict={inputs: input_sentence})
predict_sentence[0]
idx = sess.run(tf.argmax(predict_sentence,1))
if idx[0] == 1:
    print('The comment is Negative')
else:
    print('The comment is Positive')
predict_sentence = sess.run(predict, feed_dict={inputs: input_sentence})
idx = sess.run(tf.argmax(predict_sentence,1))
idx[0]
sess.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1360756206502545, 'fraction_numerical': 0.020359405837748, 'mean_word_length': 3.768697374938088, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '614033', 'license': 'MIT', 'n_tokens_mistral': 3577, 'n_tokens_neox': 3366, 'n_words': 747, 'path': '.ipynb_checkpoints/Mine-checkpoint.ipynb', 'repo_name': 'Dreamer179/Voice-classification', 'size': '9627'}