<jupyter_start><jupyter_text>A neural network approachThe ngram model achieved reasonable results, but there are several pitfalls of ngrams that are hard to get around. For small n values, ngram models don't keep enough history of the word. For large n values, the model can't make proper inductions, and will just repeat words from the training corpus.A neural network can be a good way to work through a word, remembering what is important and forgetting what is not. Since words are of variable length, it is good to use a recurrent neural network, which can iterate through a word.  Imports<jupyter_code>import random
import numpy as np
from sklearn.model_selection import train_test_split
import keras
from keras.models import Sequential
from keras.layers import Dense, Activation, SimpleRNN, LSTM
from keras.preprocessing.sequence import pad_sequences
from keras.optimizers import RMSprop
import matplotlib.pyplot as plt

#pathnames
cities_file = 'city-names.txt'

#import cities
with open(cities_file) as file:
    cities = [line.strip() for line in file.readlines()]<jupyter_output><empty_output><jupyter_text>Converting words to NN inputsWe start by tokenising city names into lists of characters<jupyter_code>def tokenize(name):
    """Turn a string into a list of character tokens."""
    #name = name.lower() #ignore cases
    characters = [c for c in name] #make a list of characters
    characters = ['<START>'] + characters + ['<STOP>'] #add beginning and end token
    return characters

tokenized_cities = [tokenize(city) for city in cities]
#make a list of all characters
characters = sorted(list(set(char for city in tokenized_cities for char  in city)))

def char_to_vec(char):
    """Conversion from a character to a one-hot vector"""
    i = characters.index(char)
    v = np.zeros(len(characters))
    v[i] = 1
    return v
maxlen = max(len(city) for city in tokenized_cities)

#converting words to input and output matrices

def tokens_to_input(tokens):
    """Create list of inputs """
    in_tokens = tokens[:-1] #exclude the stop symbol
    inputs = list()
    for i in range(len(in_tokens)):
        padding = [np.zeros(len(characters)) for i in range(maxlen - (i + 1))]
        vectors = [char_to_vec(char) for char in in_tokens[:i]]
        matrix = np.stack(padding + vectors)
        inputs.append(matrix)
        
    return inputs
    
def tokens_to_output(tokens):
    """Create the list of output vectors for a word"""
    out_tokens = tokens[1:] #exclude the start symbol and shift everything 1 step to the front
    vectors = [char_to_vec(char)for char in out_tokens]
    return vectors
# generate the input and target output presentations
input_stacks = [tokens_to_input(tokens) for tokens in tokenized_cities]
output_stacks = [tokens_to_output(tokens) for tokens in tokenized_cities]

#train test split
cities_train, cities_val, X_train_raw, X_val_raw, Y_train_raw, Y_val_raw = train_test_split(cities, input_stacks, output_stacks)

#unfold input and output stacks
def unfold_stacks(data):
    return [array for city_arrays in data for array in city_arrays]

X_train_raw = unfold_stacks(X_train_raw)
X_val_raw = unfold_stacks(X_val_raw)
Y_train_raw = unfold_stacks(Y_train_raw)
Y_val_raw = unfold_stacks(Y_val_raw)<jupyter_output><empty_output><jupyter_text>Generate batches<jupyter_code>X_train = pad_sequences(X_train_raw, dtype = 'float')
Y_train = pad_sequences(Y_train_raw, dtype = 'float')
X_val = pad_sequences(X_val_raw, dtype = 'float')
Y_val = pad_sequences(Y_val_raw, dtype = 'float')
print(X_train.shape)
print(Y_train.shape)
print(X_val.shape)
print(Y_val.shape)<jupyter_output>(2332, 18, 49)
(2332, 49)
(770, 18, 49)
(770, 49)<jupyter_text>Defining the network<jupyter_code>#defining some values

#model dimensions
features = len(characters)
hidden_size = 20

#training
batch_size = 30
epochs = 40
model = Sequential([LSTM(units = hidden_size, activation = 'relu', recurrent_initializer='identity', bias_initializer='zeros'),
                    Dense(units = features),
                    Activation('softmax')])<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>optimizer = RMSprop(learning_rate=0.01)
model.compile(loss='categorical_crossentropy',
              optimizer=optimizer,
              metrics=['accuracy'])

history = model.fit(X_train, Y_train, 
                    batch_size = batch_size, 
                    epochs=epochs,
                    validation_data=(X_val, Y_val),
                    verbose = 2)
model.summary()
#plot training history

fig, ax1 = plt.subplots()


color = 'cornflowerblue' 
ax1.plot(history.history['loss'], label = 'training loss', linestyle = '--', color = color)
ax1.plot(history.history['val_loss'], label = 'valuation loss', color = color)
ax1.tick_params(axis='y', labelcolor=color)
ax1.set_ylabel('loss')

ax2 = ax1.twinx()

color = 'coral'
ax2.plot(history.history['accuracy'], label = 'training accuracy', linestyle = '--', color = color)
ax2.plot(history.history['val_accuracy'], label = 'valuation accuracy', color = color)
ax2.tick_params(axis='y', labelcolor=color)
ax2.set_ylabel('accuracy')

#fig.legend()
print(model.evaluate(X_val, Y_val))
print(model.evaluate(X_train, Y_train))<jupyter_output>770/770 [==============================] - 0s 60us/step
[2.541330257948343, 0.26103895902633667]
2332/2332 [==============================] - 0s 50us/step
[2.4154292093541976, 0.28130361437797546]<jupyter_text>Generate city names<jupyter_code>def output_to_char(vector):
    choice = np.random.choice(characters, p = vector)
    return choice

def start_matrix():
    vec = char_to_vec('<START>')
    matrix = np.expand_dims(vec, axis=0)
    return matrix
    
def predict_next(model, word):
    batch = pad_sequences(word, maxlen = maxlen - 1)
    prediction = model.predict(batch)
    predicted_char = output_to_char(prediction[0])
    return predicted_char
    
def generate_word(model):
    word = start_matrix()
    
    #keep generating new characters    
    finished = False
    
    while not finished:
        next_char = predict_next(model, [word])
        
        #signal stop when the last character was a stop symbol
        if next_char == '<STOP>':
            finished = True
        
        new_vec = np.expand_dims(char_to_vec(next_char), axis = 0)
        word = np.vstack([word, new_vec])
    
    #convert word to string
    indices = [np.nonzero(row)[0][0] for row in word]
    tokens = [characters[i] for i in indices]
    string = '' .join(tokens[1:-1])

    return string
for i in range(10):
    print(generate_word(model))<jupyter_output>asbdk
eborna
eeoeeebue
yesasl
over
nsele
ule
eprsce
adlltrhvitnmn
lraglnhn
{'fraction_non_alphanumeric': 0.1094288712336981, 'fraction_numerical': 0.022635287063408786, 'mean_word_length': 4.333333333333333, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '434884', 'license': 'MIT', 'n_tokens_mistral': 2281, 'n_tokens_neox': 2146, 'n_words': 605, 'path': 'RNN.ipynb', 'repo_name': 'lukavdplas/historic-dutch-city-names', 'size': '6671'}