<jupyter_start><jupyter_text>코드 출처:    https://github.com/keras-team/keras/blob/4b9ba1b3fa99755910971bd75a87f3d6824d2be8/examples/pretrained_word_embeddings.py GloVe embedding data can be found at:  http://nlp.stanford.edu/data/glove.6B.zip  (source page: http://nlp.stanford.edu/projects/glove/)  20 Newsgroup data can be found at:  http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html  20-newsgroups데이터  설명This data set is a collection of 20,000 messages, collected from 20 different netnews newsgroups. One thousand messages from each of the twenty newsgroups were chosen at random and partitioned by newsgroup name. The list of newsgroups from which the messages were chose is as follows:   glove.6B  임베딩 데이터 설명Wikipedia 2014 + Gigaword 5 (6B tokens, 400K vocab, uncased, 50d, 100d, 200d, & 300d vectors, 822 MB download):<jupyter_code>from __future__ import print_function

import os
import sys
import numpy as np
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
from keras.layers import Dense, Input, GlobalMaxPooling1D
from keras.layers import Conv1D, MaxPooling1D, Embedding
from keras.models import Model
print('keras'+keras.__version__)<jupyter_output>keras2.1.4<jupyter_text>Preparing the text data<jupyter_code>BASE_DIR = './'
GLOVE_DIR = os.path.join('./glove.6B')
TEXT_DATA_DIR = os.path.join('./20-newsgroups')<jupyter_output><empty_output><jupyter_text>first, build index mapping words in the embeddings setto their embedding vector<jupyter_code>print('Indexing word vectors.')<jupyter_output>Indexing word vectors.<jupyter_text>Preparing the Embedding layer first, build index mapping words in the embeddings set to their embedding vector<jupyter_code>embeddings_index = {}
f = open(os.path.join('./glove.6B/glove.6B.100d.txt'))
for line in f:
    values = line.split()
    word = values[0]
    coefs = np.asarray(values[1:], dtype='float32')
    embeddings_index[word] = coefs
f.close()

print('Found %s word vectors.' % len(embeddings_index))
#second, prepare text samples and their labels
print('Processing text dataset')
texts = []  # list of text samples
labels_index = {}  # dictionary mapping label name to numeric id
labels = []  # list of label ids
for name in sorted(os.listdir(TEXT_DATA_DIR)):
    path = os.path.join(TEXT_DATA_DIR, name)
    label_id = len(labels_index)
    labels_index[name] = label_id
    f = open(path, encoding='latin-1')
    t = f.read()
    i = t.find('\n\n')  # skip header
    if 0 < i:
        t = t[i:]
    texts.append(t)
    f.close()
    labels.append(label_id)

print('Found %s texts.' % len(texts))


# texts = []  # list of text samples
# labels_index = {}  # dictionary mapping label name to numeric id
# labels = []  # list of label ids
# for name in sorted(os.listdir(TEXT_DATA_DIR)):
#     path = os.path.join(TEXT_DATA_DIR, name )
#     if os.path.isdir(path):
#         label_id = len(labels_index)
#         labels_index[name] = label_id
#         for fname in sorted(os.listdir(path)):
#             if fname.isdigit():
#                 fpath = os.path.join(path, fname)
#                 args = {} if sys.version_info < (3,) else {'encoding': 'latin-1'}
#                 with open(fpath, **args) as f:
#                     t = f.read()
#                     i = t.find('\n\n')  # skip header
#                     if 0 < i:
#                         t = t[i:]
#                     texts.append(t)
#                 labels.append(label_id)

# print('Found %s texts.' % len(texts))<jupyter_output>Found 21 texts.<jupyter_text>Latin-1은 ISO-8859-1과 같은 인코딩시스템이다. 유니코드와 관련된 자세한 내용은 https://docs.python.org/3.4/howto/unicode.html 참고<jupyter_code># finally, vectorize the text samples into a 2D integer tensor
tokenizer = Tokenizer(num_words=MAX_NUM_WORDS)
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)

word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)

labels = to_categorical(np.asarray(labels))
print('Shape of data tensor:', data.shape)
print('Shape of label tensor:', labels.shape)
# split the data into a training set and a validation set
indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = labels[indices]
num_validation_samples = int(VALIDATION_SPLIT * data.shape[0])
x_train = data[:-num_validation_samples]
y_train = labels[:-num_validation_samples]
x_val = data[-num_validation_samples:]
y_val = labels[-num_validation_samples:]

print('Preparing embedding matrix.')
# prepare embedding matrix
num_words = min(MAX_NUM_WORDS, len(word_index) + 1)
embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))
for word, i in word_index.items():
    if i >= MAX_NUM_WORDS:
        continue
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        # words not found in embedding index will be all-zeros.
        embedding_matrix[i] = embedding_vector

# load pre-trained word embeddings into an Embedding layer
# note that we set trainable = False so as to keep the embeddings fixed
embedding_layer = Embedding(num_words,
                            EMBEDDING_DIM,
                            weights=[embedding_matrix],
                            input_length=MAX_SEQUENCE_LENGTH,
                            trainable=False)

print('Training model.')

# train a 1D convnet with global maxpooling
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Conv1D(128, 5, activation='relu')(embedded_sequences)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = GlobalMaxPooling1D()(x)
x = Dense(128, activation='relu')(x)
preds = Dense(len(labels_index), activation='softmax')(x)

model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
              optimizer='rmsprop',
              metrics=['acc'])

model.fit(x_train, y_train,
          batch_size=128,
          epochs=10,
          validation_data=(x_val, y_val))<jupyter_output>Train on 17 samples, validate on 4 samples
Epoch 1/10
17/17 [==============================] - 1s 38ms/step - loss: 3.0825 - acc: 0.0588 - val_loss: 4.2982 - val_acc: 0.0000e+00
Epoch 2/10
17/17 [==============================] - 0s 8ms/step - loss: 2.9439 - acc: 0.0588 - val_loss: 4.9330 - val_acc: 0.0000e+00
Epoch 3/10
17/17 [==============================] - 0s 8ms/step - loss: 2.9640 - acc: 0.0588 - val_loss: 4.8590 - val_acc: 0.0000e+00
Epoch 4/10
17/17 [==============================] - 0s 8ms/step - loss: 2.7087 - acc: 0.0588 - val_loss: 4.2378 - val_acc: 0.0000e+00
Epoch 5/10
17/17 [==============================] - 0s 8ms/step - loss: 2.4235 - acc: 0.2353 - val_loss: 5.2631 - val_acc: 0.0000e+00
Epoch 6/10
17/17 [==============================] - 0s 8ms/step - loss: 2.0829 - acc: 0.5882 - val_loss: 5.1953 - val_acc: 0.0000e+00
Epoch 7/10
17/17 [==============================] - 0s 8ms/step - loss: 2.0199 - acc: 0.2941 - val_loss: 5.6543 - val_acc: 0.0000e+00
Epoch 8/10
17/17 [[...]
{'fraction_non_alphanumeric': 0.14612822647793505, 'fraction_numerical': 0.05023591451568138, 'mean_word_length': 3.7104575163398694, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '547305', 'license': 'MIT', 'n_tokens_mistral': 2763, 'n_tokens_neox': 2529, 'n_words': 612, 'path': 'glove/Glove_keras.ipynb', 'repo_name': 'songys/CS224d_17_KOR_SUB', 'size': '7206'}