<commit_before>from common import weighted_choice,get_documents,get_corpus

from pprint import pprint
from heapq import nlargest
from time import time
from random import randint

K = 3
ITERS = 1000
alpha = 0.1 # document alpha factor
eta = 0.1 # word eta factor
MULTIPLICATION = 10

documents = get_documents(MULTIPLICATION)
corpus = get_corpus(documents)
topics = list(range(1,K+1))
docs = dict(enumerate(documents))

# INITIALIZE

t_wd = {w:{} for w in corpus}
n_dt = {d:{t:0 for t in topics} for d in docs}
n_wt = {w:{t:0 for t in topics} for w in corpus}
n_t = {t:0 for t in topics}

# randomly select topic for each word in each document
for d in docs:
	for w in docs[d]:
		t = randint(1,K)
		t_wd[w][d] = t
		n_dt[d][t] += 1
		n_wt[w][t] += 1
		n_t[t] += 1

# MAIN LOOP
t0=time()
for _ in range(ITERS):
	for d in docs:
		denom_dt = len(docs[d])-1 + K*alpha # TODO check
		for w in docs[d]:
			# decr current w/d/t
			t = t_wd[w][d]
			n_dt[d][t] -= 1
			n_wt[w][t] -= 1
			n_t[t] -= 1
			
			p_t = {}
			for t in topics:
				denom_wt = n_t[t] + len(corpus)*eta # TODO check len(corpus)???
				p_wt = 1.* (n_wt[w][t] + eta)   / denom_wt
				p_dt = 1.* (n_dt[d][t] + alpha) / denom_dt
				p_t[t] = p_wt * p_dt

			# incr current w/d/t
			t = weighted_choice(p_t)
			t_wd[w][d] = t
			n_dt[d][t] += 1
			n_wt[w][t] += 1
			n_t[t] += 1
t1=time()

# RESULTS

for t in topics:
	top = nlargest(7,((w,n_wt[w][t]) for w in n_wt),key=lambda x:x[1])
	print('topic #{}  {}'.format(t,' '.join([x[0] for x in top if x[1]>0])))

print('{0:.2f} iters per second'.format(ITERS/(t1-t0)))
<commit_msg>lda denom fix<commit_after>from common import weighted_choice,get_documents,get_corpus

from pprint import pprint
from heapq import nlargest
from time import time
from random import randint

K = 3
ITERS = 1000
alpha = 0.1 # document alpha factor
eta = 0.1 # word eta factor
MULTIPLICATION = 10

documents = get_documents(MULTIPLICATION)
corpus = get_corpus(documents)
topics = list(range(1,K+1))
docs = dict(enumerate(documents))

# INITIALIZE

t_wd = {w:{} for w in corpus}
n_dt = {d:{t:0 for t in topics} for d in docs}
n_wt = {w:{t:0 for t in topics} for w in corpus}
n_t = {t:0 for t in topics}
n_w = {w:0 for w in corpus} # DF

# randomly select topic for each word in each document
for d in docs:
	for w in docs[d]:
		t = randint(1,K)
		t_wd[w][d] = t
		n_dt[d][t] += 1
		n_wt[w][t] += 1
		n_t[t] += 1
		n_w[w] += 1

# MAIN LOOP
t0=time()
for _ in range(ITERS):
	for d in docs:
		denom_dt = len(docs[d])-1 + K*alpha
		for w in docs[d]:
			denom_wt = n_w[w]-1 + K*eta
			
			# decr current w/d/t
			t = t_wd[w][d]
			n_dt[d][t] -= 1
			n_wt[w][t] -= 1
			n_t[t] -= 1
			
			p_t = {}
			for t in topics:
				p_wt = 1.* (n_wt[w][t] + eta)   / denom_wt
				p_dt = 1.* (n_dt[d][t] + alpha) / denom_dt
				p_t[t] = p_wt * p_dt

			# incr current w/d/t
			t = weighted_choice(p_t)
			t_wd[w][d] = t
			n_dt[d][t] += 1
			n_wt[w][t] += 1
			n_t[t] += 1
t1=time()

# RESULTS

for t in topics:
	top = nlargest(7,((w,n_wt[w][t]) for w in n_wt),key=lambda x:x[1])
	print('topic #{}  {}'.format(t,' '.join([x[0] for x in top if x[1]>0])))

print('{0:.2f} iters per second'.format(ITERS/(t1-t0)))

{'fraction_non_alphanumeric': 0.16650928548945546, 'fraction_numerical': 0.02612527541706012, 'mean_word_length': 3.306233062330623, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3663427', 'n_tokens_mistral': 1619, 'n_tokens_neox': 1435, 'n_words': 400}