<jupyter_start><jupyter_text>Modelling Chinese Word Segmentation as Sequence to Sequence Prediction Problem   Author: <NAME>Student ID: asa224 Email: <EMAIL>  Generate data for sequence to sequence modelling This returns the structure "data" which contains sentences as individual lists, along with the class label of each character.<jupyter_code>import numpy as np
import collections
from sklearn.model_selection import train_test_split
import keras
from keras.utils.np_utils import to_categorical
from keras.preprocessing import sequence
import cPickle as pickle
SPECIAL_SYMBOL = u'\u02e0'
# we open the count file, get the word and assign labels to each character
# cannot use dictionaries, since the same character may appear again and overwrites the value at its place in dict.

def generateTupleListAccToSentences(filename):
    """
    This function generates a data list of lists, which contains sequences and corresponding
    labels for each character, according to the sentences in the input file. This function 
    takes the whole training set txt file as input, and generates sequences according to the 
    line, ie. each sequence is a line. 
    
    INPUT: Input to this function is the training text file
    filename='/local-scratch/asa224/wseg_simplified_cn.txt'
    
    If you want to use this data as training data for LSTM, you have to pad the sequences 
    since they are not of the same length. 
    """
    # filepath = '/mnt/D82A1A8F2A1A6B30/wseg_simplified_cn.txt'
    with(open(filepath, 'rb')) as f:
        data = [[]]
        count = 0
        for line in f:
            line = unicode(line, 'utf-8')
            line = line.replace('\n', '')
            words = line.split(' ')

            for word in words:
                if len(word) == 1:
                    data[count].append((word[0], 3))
                else:
                    for i, character in enumerate(word):
                        if i == 0: # this is the first letter
                            data[count].append((character, 0))
                        elif i == (len(word) - 1): # this is the last letter
                            data[count].append((character, 2))
                        else: # this is somewhere in the middle
                            data[count].append((character, 1))
            data.append([])
            count += 1

        f.close()
        
        return data
    
def generateWordFile(filename, output_dir):
    """
    The function generates the word file, similar to the count_1w.txt file provided by Prof. Anoop
    
    The output of the file can be used to parse the characters, and is an input to the 
    generateTupleList() function as well. 
    
    filename='/local-scratch/asa224/wseg_simplified_cn.txt'
    
    INPUT: Input to this function is the training text file. 
    
    """
    with(open(filename, 'rb')) as f:
        word_file_1M = open(output_dir + 'word_file_1M', 'wb')
        for line in f:
            line = unicode(line, 'utf-8')
            line = line.replace('\n', '')
            line = line.split(' ')
            
            # add the newline back using a special symbol
            line.append(SPECIAL_SYMBOL)
            for word in line:
                word_file_1M.write(word.encode('utf-8') + '\t'.encode('utf-8') + str(0).encode('utf-8') +\
                                           '\n'.encode('utf-8'))
        f.close()
    word_file_1M.close()

def generateInputWordFile(filename, output_dir):
    """
    The function generates the word file, similar to the count_1w.txt file provided by Prof. Anoop
    
    THIS GENERATES WORD FILE FOR INPUT TEXT. 
    
    The output of the file can be used to parse the characters, and is an input to the 
    generateTupleList() function as well. 
    
    INPUT: Input to this function is the training text file. 
    
    filename='/local-scratch/asa224/input'
    
    """
    with(open(filename, 'rb')) as f:
        word_file_1M = open(output_dir + 'input_word_file', 'wb')
        for line in f:
            line = unicode(line, 'utf-8')
            # replace the newline character with the special unicode symbol
            line = line.replace('\n', SPECIAL_SYMBOL)
            for word in line:
                word_file_1M.write(word.encode('utf-8') + '\t'.encode('utf-8') + str(0).encode('utf-8') +\
                                           '\n'.encode('utf-8'))
        f.close()
    word_file_1M.close()

def generateTupleList(filename):
    """
    This function is similar to the above function in the sense that it assigns labels to each
    character in the training set. The function returns a list of tuples, in which each tuple
    contains a single character and its corresponding label. 
    
    INPUT: Input to this function is a WORD FILE generated by generateWordFile(filename) function. 
    
    filename='/local-scratch/asa224/word_file_1M'
    
    Use this function in conjunction with nGramSequenceGenerator(labelledlist, n) to create a
    training set with constant sequence size, which does not require paddings. 
    """
    with(open(filename, 'rb')) as f:
        label = []
        for line in f:
            word, count = line.split('\t')
            # making sure the parsing is going fine
            assert int(count) == 0

            word = unicode(word, 'utf-8')
            if len(word) == 1:
                label.append((word[0], 3))
            else:
                for i, character in enumerate(word):
                    if i == 0: # this is the first letter
                        label.append((character, 0))
                    elif i == (len(word) - 1): # this is the last letter
                        label.append((character, 2))
                    else: # this is somewhere in the middle
                        label.append((character, 1))

        f.close()
        return label
    
def nGramSequenceGenerator(labelledlist, n):
    """
    Takes as input the label list of tuples generated by the code above. 
    The function generates sequence of size "n" from the given list. 
    """
    count = len(labelledlist)/n
    ngrammedlist = []
    for i in range(count):
        ngrammedlist.append( labelledlist[i*n : (i+1)*n])
    return ngrammedlist
parent_path = '/home/asa224/Desktop/students_less_asa224/Test Folder on Less/'
useCache = False
n_timesteps = 13

if useCache == False:
    print('Generating new word file with new newline unicode encoding..')
    generateWordFile(filename=parent_path + 'wseg_simplified_cn.txt', output_dir=parent_path)
    print('Generating list of tuples of all characters in the word file')
    list_of_tuples = generateTupleList(filename=parent_path + 'word_file_1M')
    print('Generating ngram sequences, with n_timesteps = {}'.format(n_timesteps))
    data = nGramSequenceGenerator(list_of_tuples, n=n_timesteps)
    
#     pickle.dump( list_of_tuples, open( parent_path + "list_of_tuples_"+str(n_timesteps)+".p", "wb" ) )
#     pickle.dump( data, open( parent_path + "data_"+str(n_timesteps)+".p", "wb" ) )
    
else:
    list_of_tuples = pickle.load( open( parent_path + "list_of_tuples_"+str(n_timesteps)+".p", "rb" ) )
    data = pickle.load(open( parent_path + "data_"+str(n_timesteps)+".p", "rb" ))
data[0:3]<jupyter_output><empty_output><jupyter_text>Build initial integer embeddings<jupyter_code>all_chars = [data[i][j][0] for i in range(0, len(data)) for j in range(0, len(data[i]))]
print('Total number of characters in the dataset: {}'.format(len(all_chars)))
def build_dataset(words):
    count = collections.Counter(words).most_common()
    dictionary = dict()
    for word, _ in count:
        dictionary[word] = len(dictionary)
    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
    return dictionary, reverse_dictionary
print('Building dictionaries for initial embedding word -> integers...')
orig_dict, ret_dict = build_dataset(all_chars)
del all_chars
# ret_dict[orig_dict[u'\u6743']]
print('Number of unique characters in the dictionary: {}'.format(len(orig_dict)))<jupyter_output>Number of unique characters in the dictionary: 5420<jupyter_text>Save the lookup dictionaries<jupyter_code>import cPickle as pickle
pickle.dump( orig_dict, open( "./temp_files/orig_dict.p", "wb" ) )
pickle.dump( ret_dict, open( "./temp_files/ret_dict.p", "wb" ) )<jupyter_output><empty_output><jupyter_text>Create sequences suitable for training<jupyter_code>print('Creating sequences suitable for training...')
x = [[]]
y = [[]]
for i in range(0, len(data)): # iterate over the whole dataset
    for j in range(0, len(data[i])): # iterate over the current sentence
        x[i].append(orig_dict[data[i][j][0]])
        y[i].append(data[i][j][1])
    x.append([])
    y.append([])
del data
print('Splitting data to training/testing...')
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.20, shuffle=False, random_state=42)
del x,y
print('Size of the Dataset\n')
print('X_TRAIN: {}'.format(np.shape(X_train)))
print('Y_TRAIN: {}'.format(np.shape(y_train)))

print('X_TEST: {}'.format(np.shape(X_test)))
print('Y_TEST: {}'.format(np.shape(y_test)))<jupyter_output>Size of the Dataset

X_TRAIN: (2623861, 13)
Y_TRAIN: (2623861, 13)
X_TEST: (655966,)
Y_TEST: (655966,)<jupyter_text>Convert the labels to categorical.<jupyter_code>print('Converting labels to categorical...')<jupyter_output>Converting labels to categorical...<jupyter_text>This is what they look like now<jupyter_code>for i in y_train:
    print(i)
    break<jupyter_output>[0, 2, 0, 2, 3, 3, 0, 2, 0, 2, 3, 3, 0]<jupyter_text>Start conversion Testing labels There are empty lists in the test set, let's remove them.<jupyter_code>X_train = [t for t in X_train if t != []]
y_train = [t for t in y_train if t != []]

X_test = [t for t in X_test if t != []]
y_test = [t for t in y_test if t != []]<jupyter_output><empty_output><jupyter_text>Training labels<jupyter_code>for i in range(0, len(y_train)):
    y_train[i] = to_categorical(y_train[i], num_classes=4)
for i in range(0, len(y_test)):
    y_test[i] = to_categorical(y_test[i], num_classes=4)<jupyter_output><empty_output><jupyter_text>This how it looks now<jupyter_code>for i in y_train:
    print(i)
    break<jupyter_output>[[ 1.  0.  0.  0.]
 [ 0.  0.  1.  0.]
 [ 1.  0.  0.  0.]
 [ 0.  0.  1.  0.]
 [ 0.  0.  0.  1.]
 [ 0.  0.  0.  1.]
 [ 1.  0.  0.  0.]
 [ 0.  0.  1.  0.]
 [ 1.  0.  0.  0.]
 [ 0.  0.  1.  0.]
 [ 0.  0.  0.  1.]
 [ 0.  0.  0.  1.]
 [ 1.  0.  0.  0.]]<jupyter_text>Firstly, we must update the get_sequence() function to reshape the input and output sequences to be 3-dimensional to meet the expectations of the LSTM. The expected structure has the dimensions [samples, timesteps, features]. The classification problem has 1 sample (e.g. one sequence), a configurable number of timesteps, and one feature per timestep. Let's reshape the training and testing data  No need to pad sequences when working on constant size sequences<jupyter_code># # maximum length of the sequence
# maxval = len(max(X_train,key=len))
# X_train = sequence.pad_sequences(X_train, value=-1, maxlen=100)
# y_train = sequence.pad_sequences(y_train, value=-1, maxlen=100)

# X_test = sequence.pad_sequences(X_test, value=-1, maxlen=100)
# y_test = sequence.pad_sequences(y_test, value=-1, maxlen=100)
embedding = True
# reshape input and output data to be suitable for LSTMs
print('Reshaping input vectors...')
if embedding == False:
    X_train = np.array(X_train).reshape(np.shape(X_train)[0], n_timesteps, 1)
    y_train = np.array(y_train).reshape(np.shape(y_train)[0], n_timesteps, 4)

    X_test = np.array(X_test).reshape(np.shape(X_test)[0], n_timesteps, 1)
    y_test = np.array(y_test).reshape(np.shape(y_test)[0], n_timesteps, 4)
    
else:
    # reshape input and output data to be suitable for LSTMs
    X_train = np.array(X_train).reshape(np.shape(X_train)[0], n_timesteps)
    y_train = np.array(y_train).reshape(np.shape(y_train)[0], n_timesteps, 4)

    X_test = np.array(X_test).reshape(np.shape(X_test)[0], n_timesteps)
    y_test = np.array(y_test).reshape(np.shape(y_test)[0], n_timesteps, 4)<jupyter_output>Reshaping input vectors...<jupyter_text>IF you use embedding layer, use this code<jupyter_code>X_train.shape<jupyter_output><empty_output><jupyter_text>TODO: Add embedding layer TODO: Increase LSTM units TODO: Decrease the sequence size TODO: Decrease the sequence size from the training set generation itself<jupyter_code>print('Defining Model...')
from keras.models import Sequential
from keras.layers import LSTM, TimeDistributed, Dense, Masking, Embedding, Dropout
# define LSTM
model = Sequential()
model.add(Embedding(input_dim=len(orig_dict), output_dim=200, input_length=n_timesteps)) # dictionary size, embedding vector size, timesteps
model.add(LSTM(300, return_sequences=True))
model.add(LSTM(300, return_sequences=True))
model.add(Dropout(0.2))
model.add(TimeDistributed(Dense(4, activation='softmax')))
model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['acc'])
model.summary()
from keras.callbacks import ModelCheckpoint
mc = ModelCheckpoint(parent_path + 'checkpoints/' + 'model.seq2seq_nl_preview_{epoch:02d}-{val_acc:.2f}.hdf5', 
                                monitor='val_acc', 
                                verbose=0, 
                                save_best_only=False, 
                                save_weights_only=False, 
                                mode='auto', period=1)
history = model.fit(X_train, y_train, epochs=1, 
          batch_size=100, verbose=1, validation_data=[X_test, y_test], callbacks=[mc])
model.save('/home/asa224/Desktop/students_less_asa224/Test Folder on Less/model_preview.h5')<jupyter_output>Train on 2623861 samples, validate on 655965 samples
Epoch 1/1
2623861/2623861 [==============================] - 1047s - loss: 0.4215 - acc: 0.8345 - val_loss: 0.3430 - val_acc: 0.8656
{'fraction_non_alphanumeric': 0.11721954378642872, 'fraction_numerical': 0.022810678563718788, 'mean_word_length': 3.0876470588235296, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '161634', 'license': 'MIT', 'n_tokens_mistral': 4687, 'n_tokens_neox': 4409, 'n_words': 1287, 'path': 'segmenter/answer/Experiments/Seq2Seq_Prediction.ipynb', 'repo_name': 'trane293/nlp-project', 'size': '13909'}