<jupyter_start><jupyter_text>Train character embeddings using word2vec skipgram approach We look at ith character and try to predict (i-1) and (i+1) character<jupyter_code>#for inline plotting in notebooks
%matplotlib inline  
from keras.layers import Input, Dense
from keras.models import Model
import numpy as np
import matplotlib.pyplot as plt
import random<jupyter_output><empty_output><jupyter_text>Get data<jupyter_code># load data
path = "./../data/harry_potter_3.txt"
text = open(path).read()

corpus_length = len(text)

print('corpus length:', corpus_length)
unique_chars = list(set(text))
print('total no of unique chars:', len(unique_chars))

print "\nSet of characters : \n" + str(unique_chars)
def classToOneHot(item, classes):

	x = [0. for _ in classes]
	
	x[classes.index(item)] = 1.

	return x
classToOneHot('a', unique_chars)
def findClosest(labels, coords, n=3):
	def dist(p1, p2):
		return np.linalg.norm(np.array(p1)-np.array(p2))
	for label, pos in zip(labels, coords):
		print(label)
		dists = []
		for l2, p2 in zip(labels, coords):
			if l2==label: continue
			dists.append((l2, dist(pos, p2)))
		dists = sorted(dists, key=lambda x : x[1])
		if n == -1:
			if dists[0][1] >= 0.2:
				print("\t{}\t{}".format(dists[0][0], dists[0][1]))
			else:
				for d in dists:
					if d[1] <= 0.15:
						print("\t{}\t{}".format(d[0], d[1]))
		else:		
			for d in dists[:n]:
				print("\t{}\t{}".format(d[0], d[1]))
# prepare training data X, y
# both X,y are are characters 

skip_window = 1 # how much we see on either side of center word
global_index = skip_window # 

X = []
y = []

for i in range(global_index, (corpus_length - skip_window)):

    #print i
    
    X_char = text[i]
    y1_char = text[i-1]
    y2_char = text[i+1]
    
    #print X_char
    #print y1_char, y2_char
    X.append(classToOneHot(X_char, unique_chars))
    yp=[classToOneHot(y1_char, unique_chars), classToOneHot(y2_char, unique_chars)] 
    
    y.append(yp)
    
    
    
nb_cols = len(y[0])
y_cols = [np.array([v[i] for v in y], np.float32) for i in range(nb_cols)]

input_layer = Input(shape=(len(unique_chars),))
encoding = Dense(2, activation='linear')(input_layer)

output_1 = Dense(len(unique_chars), activation='softmax')(encoding)
output_2 = Dense(len(unique_chars), activation='softmax')(encoding)

model = Model(input=[input_layer], output=[output_1, output_2])#, output_3, output_4])

encoder = Model(input = [input_layer], output=[encoding])

# when using larger windows, remember to add more loss weights
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', loss_weights=[0.5, 0.5])

model.fit(X, y_cols, nb_epoch=15, batch_size=1024, shuffle=True, verbose=True)
x2 = [classToOneHot(ch, unique_chars) for ch in unique_chars[:-1]]
encodings = encoder.predict(np.array(x2))
for ch, e in zip(unique_chars[:-1], encodings):
    print("{}\t{}\t{}".format(ch, e[0], e[1]))
labels = unique_chars[:-1]
xd = [e[0] for e in encodings]
yd = [e[1] for e in encodings]
fig, ax = plt.subplots()
for i, txt in enumerate(labels):
    _ = ax.annotate(txt, (xd[i],yd[i]))


findClosest(labels, encodings, n=2)

plt.show()<jupyter_output>"	1.17302238941
	-	1.18435323238

	.	1.99439966679
	,	2.17865157127
!
	?	0.0853088349104
	"	0.42432102561
 
	O	1.68579626083
	N	1.75950944424
"
	?	0.342957913876
	!	0.42432102561
'
	7	0.184250116348
	i	0.186912685633
&
	0	0.0407764464617
	3	0.152949243784
)
	I	0.444941490889
	:	0.649912297726
(
	i	0.0699118226767
	7	0.077191285789
*
	U	0.127925857902
	A	0.16287907958
-
	E	0.3862657547
	1	0.477983534336
,
	.	0.497514098883
	;	0.564466774464
.
	,	0.497514098883
	!	0.753324687481
1
	E	0.0972627326846
	X	0.164102837443
0
	&	0.0407764464617
	3	0.113879621029
3
	0	0.113879621029
	&	0.152949243784
2
	`	0.18566018343
	'	0.263157486916
5
	4	0.0411075912416
	K	0.0954831689596
4
	5	0.0411075912416
	K	0.0544529929757
7
	i	0.0121862376109
	(	0.077191285789
6
	\	0.0557214766741
	_	0.153291583061
9
	`	0.0895241498947
	A	0.0929888635874
;
	,	0.564466774464
	)	0.872178018093
:
	e	0.527898609638
	)	0.649912297726
?
	!	0.0853088349104
	"	0.342957913876
A
	9	0.0929888635874
	`	0.135528936982
C
	S	0.11[...]
{'fraction_non_alphanumeric': 0.14296046287367406, 'fraction_numerical': 0.18659594985535197, 'mean_word_length': 4.154037267080746, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '341397', 'license': 'MIT', 'n_tokens_mistral': 2276, 'n_tokens_neox': 1840, 'n_words': 386, 'path': 'module4/CharacterEmbedding_SkipGram.ipynb', 'repo_name': 'kochhar/Representation-Learning-for-NLP', 'size': '4148'}