<jupyter_start><jupyter_text>Prepare a file to be written out to the word2vec code  Reverse dictionary<jupyter_code># Create reverse dictionary
fwddict = onehot.data_mapping
bckdict = ['onehot'] * onehot.vocab_size;
for dictkey in fwddict:
    bckdict[ fwddict[dictkey] ] = dictkey<jupyter_output><empty_output><jupyter_text>Weights of the final layer<jupyter_code># Get the final weights from the optimizer function
Wc = regressor.weights_[-1].T

# Normalize if desired
Wc /= np.linalg.norm(Wc,axis=1)[:,None]
regressor.bias_<jupyter_output><empty_output><jupyter_text>Write the final weights to file<jupyter_code>fp = open('imvecs_directnorm.txt','wb')
fp.write("%s %s\n" % (str(Wc.shape[0]), str(Wc.shape[1]))) 
for i in range(len(bckdict)):
    fp.write('%s ' % (str(bckdict[i])))
    for j in range(Wc.shape[1]):
        fp.write('%s ' %(str(Wc[i,j])))
    fp.write("\n")
fp.close()<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>image_feats, text_tags = test_dataset.get_next_batch(test_dataset.num_images)
for i in range(batch_size):
    image_feats[i, :] /= np.linalg.norm(image_feats[i, :])
true_text_tags = np.array(map(onehot.get_multiple, text_tags))

text_pred = regressor.predict(image_feats)
from oct2py import octave
octave.addpath('../evaluation/')
[precision, recall, f1score] = octave.evaluate(true_text_tags.T, text_pred.T, 5)
print precision
print recall
print f1score<jupyter_output>0.310790402936
0.200218828364
0.243541942224<jupyter_text>Check Yonas's Code  Had to rewrite top $k$ to use evaluation matrix.<jupyter_code>def get_top_k(text_corr, reverse_dictionary, topk=5):
    
    ## My new code to fit the correlation matrix instead
    indices = text_corr.argsort()[-1:-(topk+1):-1]
    return [reverse_dictionary[i] for i in indices]

    mh = MinHeap(k)
    for word in w2v_lookup:
        mh.insert(distance_func(target_vector, w2v_lookup[word]), word)
    return mh.get_result()
def calculate_precision(ground_truth, predicted):
    ground_truth_set = set(ground_truth)
    predicted_set = set(predicted)
    return float(len(ground_truth_set.intersection(predicted_set)))/len(predicted_set)

def calculate_recall(ground_truth, predicted):
    ground_truth_set = set(ground_truth)
    predicted_set = set(predicted)
    return float(len(ground_truth_set.intersection(predicted_set)))/len(ground_truth_set)


def get_pr(ground_truths, predictions):
    precisions = []
    recalls = []
    f1s = []
    for i in range(len(ground_truths)):
        precision = calculate_precision(ground_truths[i], predictions[i])
        recall = calculate_recall(ground_truths[i], predictions[i])
 
        if precision == 0 and recall == 0:
            f1 = 0
        else:
            f1 = 2*precision*recall/(precision + recall)
 
        precisions.append(precision)
        recalls.append(recall)
        f1s.append(f1)
    return np.mean(precisions), np.mean(recalls),  np.mean(f1s)

def evaluate_regressor(regressor, val_image_feats, val_text_tags, w2v_model, k=5):
    """
    Takes a regressor and returns the precision/recall on the test data
    Args:
        regressor: a tensorflow.contrib.learn regression estimator
        val_image_feats: Image features to test performance on
        val_text_tags: Text Tags to test performance on
        w2v_model: a dictionary like object where the keys are words and the values are word vectors
        k: Top number of items to retrieve to test precision/recall on

    Returns:
        (precision, recall, f1): A tuple of precision, recall, and F1 score
    """
    val_pred = regressor.predict(val_image_feats)
    topks = []
    for i in range(val_pred.shape[0]):
        topk = get_top_k(val_pred[i, :]/np.linalg.norm(val_pred[i, :]), w2v_model, k)        
        topks.append(topk)

    return get_pr(val_text_tags, topks)


# Create reverse dictionary
fwddict = onehot.data_mapping
bckdict = ['onehot'] * onehot.vocab_size;
for dictkey in fwddict:
    bckdict[ fwddict[dictkey] ] = dictkey

evaluate_regressor(regressor, image_feats, text_tags, bckdict)
regressor.weights_[2].shape
onehot.create_data_mapping?<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11261478975350411, 'fraction_numerical': 0.01764137264378927, 'mean_word_length': 4.46042216358839, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '401165', 'license': 'Apache-2.0', 'n_tokens_mistral': 1539, 'n_tokens_neox': 1461, 'n_words': 312, 'path': 'attalos/imgtxt_algorithms/TestTFR.ipynb', 'repo_name': 'karllab41/attalos', 'size': '4138'}