<commit_before>from __future__ import division

import numpy as np
from collections import Counter

from util import (load_line_corpus, vectorize_docs)
from lda import train

docs, vocab = vectorize_docs(load_line_corpus('data/nips-2014.dat'))

K = 20
V = len(vocab)

# initialize alpha
alpha = np.ones(K)
alpha /= alpha.size


################
# Initialize beta
# Seeded method proposed by the paper
################

beta = np.zeros((K, V))

n_sample_doc = 2
doc_ids = np.random.permutation(docs.size)[: n_sample_doc * K]
for i in xrange(K):
    ids = doc_ids[n_sample_doc * i: n_sample_doc * (i+1)]
    words = np.concatenate(docs[ids])
    counter = Counter(words)
    for id_, c in counter.items():
        beta[i, id_] = c + 0.1
    beta[i] += (np.random.rand(V) * 0.2)
    beta[i] /= beta[i].sum()

assert np.abs(beta.sum(axis=1) - np.ones(K)).max() < 1e-5,\
    beta.sum(axis=1)<commit_msg>trivial commit<commit_after>from __future__ import division

import numpy as np
from collections import Counter

from util import (load_line_corpus, vectorize_docs)
from lda import train

docs, vocab = vectorize_docs(load_line_corpus('data/nips-2014.dat'))

K = 10
V = len(vocab)

# initialize alpha
alpha = np.ones(K)
alpha /= alpha.size


################
# Initialize beta
# Seeded method proposed by the paper
################

beta = np.zeros((K, V))

n_sample_doc = 2
doc_ids = np.random.permutation(docs.size)[: n_sample_doc * K]
for i in xrange(K):
    ids = doc_ids[n_sample_doc * i: n_sample_doc * (i+1)]
    words = np.concatenate(docs[ids])
    counter = Counter(words)
    for id_, c in counter.items():
        beta[i, id_] = c + 0.1
    beta[i] += (np.random.rand(V) * 0.2)
    beta[i] /= beta[i].sum()

assert np.abs(beta.sum(axis=1) - np.ones(K)).max() < 1e-5,\
    beta.sum(axis=1)
{'fraction_non_alphanumeric': 0.17362270450751252, 'fraction_numerical': 0.017807456872565387, 'mean_word_length': 3.9668508287292816, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2838433', 'n_tokens_mistral': 703, 'n_tokens_neox': 674, 'n_words': 166}