<jupyter_start><jupyter_text>Пример генерации текста рекурентной сетью<jupyter_code>import sys
import random
import numpy as np

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation
from tensorflow.keras.layers import LSTM
from tensorflow.keras.optimizers import Adam<jupyter_output><empty_output><jupyter_text>Загружаем исходные данные<jupyter_code>with open('/Users/aleksandr/Downloads/aml/aml_17/data/anna_karenina.txt', 'rb') as src:
    binary = src.read()
    text = binary.decode('utf8').lower()
print('corpus length:', len(text))<jupyter_output>corpus length: 1710408<jupyter_text>Векторизуем текстовый корпус<jupyter_code>chars = sorted(list(set(text)))
print('total chars:', len(chars))
char_indices = dict((c, i) for i, c in enumerate(chars))
indices_char = dict((i, c) for i, c in enumerate(chars))

# cut the text in semi-redundant sequences of maxlen characters
maxlen = 40
step = 3
sentences = []
next_chars = []
for i in range(0, len(text) - maxlen, step):
    sentences.append(text[i: i + maxlen])
    next_chars.append(text[i + maxlen])
print('nb sequences:', len(sentences))

print('Vectorization...')
X = np.zeros((len(sentences), maxlen, len(chars)), 
             dtype=np.bool)

y = np.zeros((len(sentences), len(chars)), dtype=np.bool)

for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        X[i, t, char_indices[char]] = 1
    y[i, char_indices[next_chars[i]]] = 1<jupyter_output>total chars: 93
nb sequences: 570123
Vectorization...<jupyter_text>Строим модель<jupyter_code># build the model: a single LSTM
print('Build model...')
model = Sequential()
model.add(LSTM(128, input_shape=(maxlen, len(chars))))
model.add(Dense(len(chars)))
model.add(Activation('softmax'))

optimizer = Adam(lr=0.01)
model.compile(loss='categorical_crossentropy',
              optimizer=optimizer)<jupyter_output>Build model...<jupyter_text>Возвращает индекс выбранного символа для вектора вероятностей<jupyter_code>def sample(preds, temperature = 1.0):
    # helper function to sample an index from a probability array
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds + 1e-32) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)<jupyter_output><empty_output><jupyter_text>Запускаем обучение<jupyter_code>for iteration in range(1, 60):
    print()
    print('-' * 50)
    print('Iteration', iteration)
    model.fit(X, y,
              batch_size=128,
              epochs=1)

    start_index = random.randint(0, len(text) - maxlen - 1)

    for diversity in [0.2, 0.5, 1.0, 1.2]:
        print()
        print('----- diversity:', diversity)

        generated = ''
        sentence = text[start_index: start_index + maxlen]
        generated += sentence
        print('----- Generating with seed: "' + sentence + '"')
        sys.stdout.write(generated)

        for i in range(400):
            x = np.zeros((1, maxlen, len(chars)))
            for t, char in enumerate(sentence):
                x[0, t, char_indices[char]] = 1.

            preds = model.predict(x, verbose=0)[0]
            next_index = sample(preds, diversity)
            next_char = indices_char[next_index]

            generated += next_char
            sentence = sentence[1:] + next_char

            sys.stdout.write(next_char)
            sys.stdout.flush()
        print()
# остановлена итерация, т.к. обучение будет долгим. Далее предсказания на предобученной модели.<jupyter_output><empty_output><jupyter_text>Предсказания на предобученной модели<jupyter_code># model.load_weights('./text-lstm.hd5')
model.load_weights('/Users/aleksandr/Downloads/aml/aml_17/text-lstm.hd5')
start_index = random.randint(0, len(text) - maxlen - 1)

for diversity in [0.2, 0.5, 1.0, 1.2]:
    print()
    print('----- diversity:', diversity)

    generated = ''
    sentence = text[start_index: start_index + maxlen]
    generated += sentence
    print('----- Generating with seed: "' + sentence + '"')
    sys.stdout.write(generated)

    for i in range(400):
        x = np.zeros((1, maxlen, len(chars)))
        for t, char in enumerate(sentence):
            x[0, t, char_indices[char]] = 1.

        preds = model.predict(x, verbose=0)[0]
        next_index = sample(preds, diversity)
        next_char = indices_char[next_index]

        generated += next_char
        sentence = sentence[1:] + next_char

        sys.stdout.write(next_char)
        sys.stdout.flush()
    print()
pass<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1351996527777778, 'fraction_numerical': 0.01953125, 'mean_word_length': 3.5319567354965584, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '163620', 'license': 'MIT', 'n_tokens_mistral': 1666, 'n_tokens_neox': 1615, 'n_words': 341, 'path': 'Lectures notebooks/(Lectures notebooks) netology Machine learning/18. Recurrent networks in image processing tasks/008-text-lstm_copy.ipynb', 'repo_name': 'Alex110117/data_analysis', 'size': '4608'}