<jupyter_start><jupyter_text>Variables<jupyter_code>data = np.load('/local_data/alexnet-feats/iaprtc_alexfc7.npz')
D = open('/local_data/alexnet-feats/iaprtc_dictionary.txt').read().splitlines()
train_ims = [ im.split('/')[-1] for im in open('/local_data/alexnet-feats/iaprtc_trainlist.txt').read().splitlines() ]
test_ims = [ im.split('/')[-1] for im in open('/local_data/alexnet-feats/iaprtc_testlist.txt').read().splitlines() ]
xTr = data['xTr'].T
yTr = data['yTr'].T
xTe = data['xTe'].T
yTe = data['yTe'].T
wc = yTr.sum(axis=0)+0.00

# Read w2vec
output_dict = {}
for i, line in enumerate(open('/work/attalos/models/glove.42B.300d.txt')):
    first_word = line[:line.find(' ')]
    if first_word in D:
        line = line.strip().split(' ')
        output_dict[line[0]] = np.array([ float(j) for j in line[1:]])
    

imfeatsize = xTr.shape[1]
vocabsize = yTr.shape[1]
wordvecsize = output_dict[output_dict.keys()[0]].shape[0]
BATCH_SIZE = 256
HIDDEN_SIZE = 300
RATE = .1<jupyter_output><empty_output><jupyter_text>For a single feature, get the related training vectorsUsage:  (im, veclist) = get_training_vectors( index_number ) For multiple features (in the list of indices), for each image feature, sum the training vectors<jupyter_code># Given an index into the corpus, get the corresponding image feature and all word vectors from it
def get_training_vectors(index):
    yTrList = []
    for i in np.nonzero( yTr[index] )[0]:
        yTrList.append( output_dict[ D[i] ] )
    return (xTr[index], np.array(yTrList))

def get_training_batch(indices):
    
    feats = []#np.zeros((len(indices),imfeatsize))
    labels = []#np.zeros((len(indices),wordvecsize))
    for j, i in enumerate(indices):
        #feats[j, :], wordfeats = get_training_vectors(i)
        feat, wordfeats = get_training_vectors(i)
        feats.append(feat)
        #labels[j, :] = wordfeats.sum(axis=0)
        labels.append(wordfeats.sum(axis=0))
    return (feats, labels)
# Given an index into the corpus, get the corresponding image feature and all word vectors from it
def get_test_vectors(index):
    yTeList = []
    for i in np.nonzero( yTe[index] )[0]:
        yTeList.append( output_dict[ D[i] ] )
    return (xTe[index], np.array(yTeList))

def get_test_batch(indices):
    
    feats = []#np.zeros((len(indices),imfeatsize))
    labels = []#np.zeros((len(indices),wordvecsize))
    for j, i in enumerate(indices):
        #feats[j, :], wordfeats = get_training_vectors(i)
        feat, wordfeats = get_test_vectors(i)
        feats.append(feat)
        #labels[j, :] = wordfeats.sum(axis=0)
        labels.append(wordfeats.sum(axis=0))
    return (feats, labels)
print imfeatsize
# Linear Regression graph
net = tflearn.input_data(shape=[None, imfeatsize])
net = tflearn.layers.core.fully_connected(net, 300, activation='sigmoid')
net = tflearn.layers.core.fully_connected(net, 300, activation='sigmoid')
#net = tflearn.layers.core.fully_connected(net, 300)
net = tflearn.regression(net, optimizer='adam', loss='binary_crossentropy')

model = tflearn.DNN(net, tensorboard_verbose=3, tensorboard_dir='/tmp/tensorboard/')
# loss_over_time = []

# # print("\nTest prediction for y = 3.2 and y = 4.5:")


    # print(m.predict([3.2, 4.5]))
imbatch, wordbatch = get_training_batch(np.arange(0, yTr.shape[0]))
imbatch_te, wordbatch_te = get_test_batch(np.arange(0, yTe.shape[0]))
model.fit(imbatch, wordbatch, validation_set=(imbatch_te, wordbatch_te), batch_size=128)
# batch_size = 10000
# for i in range(5):
#     print(i)
#     # Get a batch for training
#     imbatch, wordbatch = get_training_batch(np.random.randint(0, yTr.shape[0], batch_size))
#     #print(len(imbatch), len(imbatch[0]))
#     model.fit(imbatch, wordbatch)
#     #print("\nRegression result:")
#     #print("Y = " + str(model.get_weights(linear.W)) +
#     #  ".X + " + str(model.get_weights(linear.b)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1578542094455852, 'fraction_numerical': 0.016940451745379878, 'mean_word_length': 4.450349650349651, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '338093', 'license': 'Apache-2.0', 'n_tokens_mistral': 1473, 'n_tokens_neox': 1423, 'n_words': 279, 'path': 'attalos/imgtxt_algorithms/regress2sum/tfrun.ipynb', 'repo_name': 'karllab41/attalos', 'size': '3896'}