<commit_before>#! /usr/bin/env python
from __future__ import division

import gettext
_ = gettext.gettext
import dbconf
import time
from utils import DBM

def main():
	""" Main loop of top-news """
	cursor = DBM.cursor()
	cursor.execute("select id, name from subs where enabled = 1")
	for row in cursor:
		do_site(row[0], row[1])

def do_site(site_id, site):
	""" Process a given site """
	links = {}
	cursor = DBM.cursor()
	query = """
		select link_id, link_uri,
			unix_timestamp(now()) - unix_timestamp(link_date)
		from links, sub_statuses
		where sub_statuses.id = %s
			and status = 'published'
			and date > date_sub(now(), interval 24 hour)
			and link = link_id
			and link_votes/20 > link_negatives
		order by link_date desc
	"""
	cursor.execute(query, (site_id,))
	if cursor.rowcount < 4:
		return

	links_total = 0
	for link_id, link_uri, old in cursor:
		links_total += 1
		values = {}
		values['uri'] = link_uri
		# How old in seconds
		values['old'] = old
		values['w'] = 0
		values['c'] = 0
		values['v'] = 0
		values['links_order'] = links_total
		links[link_id] = values

	if not links_total:
		return

	links_format = ','.join(['%s'] * len(links))
	query = """
		select vote_link_id,
			sum((1-(unix_timestamp(now())
					- unix_timestamp(vote_date))/36000)) as x,
			count(*)
		from votes
		where vote_link_id in (%s)
			and vote_type='links'
			and vote_date > date_sub(now(), interval 24 hour)
			and vote_user_id > 0
			and vote_value >= 2
		group by vote_link_id
		order by x desc
	"""  % links_format
	cursor.execute(query, tuple(links))
	votes_total = 0
	votes_links = 0
	v_total = 0
	v_list = {}
	for link_id, old, votes in cursor:
		votes_links += 1
		votes_old = float(old)
		links[link_id]['v'] = votes_old
		v_total += votes_old
		v_list[link_id] = votes_old
		links[link_id]['votes'] = votes
		votes_total += votes
		links[link_id]['votes_order'] = votes_links

	if not votes_links:
		return

	v_average = v_total/votes_links
	votes_average = votes_total/votes_links

	query = """
		select comment_link_id,
			sum(1.5*(1-(unix_timestamp(now())
						- unix_timestamp(comment_date))/36000)),
			count(*)
		from comments
		where comment_link_id in (%s)
			and comment_date > date_sub(now(), interval 24 hour)
		group by comment_link_id
	""" % links_format
	cursor.execute(query, tuple(links))
	comments_total = 0
	comments_links = 0
	c_total = 0
	c_list = {}
	for link_id, old, count in cursor:
		comment_old = float(old)
		comments_links += 1
		links[link_id]['c'] = comment_old
		c_total += comment_old
		c_list[link_id] = comment_old
		links[link_id]['comments'] = count
		comments_total += count

	if not comments_links:
		return

	c_average = c_total/comments_links
	comments_average = comments_total/comments_links
	query = """
		select id, counter from link_clicks where id in (%s)
	""" % links_format

	cursor.execute(query, tuple(links))
	for link_id, clicks in cursor:
		links[link_id]['clicks'] = clicks

	cursor.close()

	print "*******************************************************************************************************"
	print "****", time.strftime("%d/%m/%Y, %H:M:S (%Z)"), "**** SITE:", site, ", Votes average:", votes_average, v_average, \
			", Comments average:", comments_average, c_average

	for link_id, link_value in links.items():
		if link_value['c'] > 0 \
				and link_value['v'] > 0 \
				and 'clicks' in link_value:
			links[link_id]['w'] = (1 - link_value['old']/(1.5*86400)) * (link_value['v'] \
						   + link_value['c'] \
						   + link_value['clicks'] * (1 - link_value['old']/86400) * 0.03)
<commit_msg>More fixes in top-news.py (2)<commit_after>#! /usr/bin/env python
from __future__ import division

import gettext
_ = gettext.gettext
import dbconf
import time
from utils import DBM

def main():
	""" Main loop of top-news """
	cursor = DBM.cursor()
	cursor.execute("select id, name from subs where enabled = 1")
	for row in cursor:
		do_site(row[0], row[1])

def do_site(site_id, site):
	""" Process a given site """
	links = {}
	cursor = DBM.cursor()
	query = """
		select link_id, link_uri,
			unix_timestamp(now()) - unix_timestamp(link_date)
		from links, sub_statuses
		where sub_statuses.id = %s
			and status = 'published'
			and date > date_sub(now(), interval 24 hour)
			and link = link_id
			and link_votes/20 > link_negatives
		order by link_date desc
	"""
	cursor.execute(query, (site_id,))
	if cursor.rowcount < 4:
		return

	links_total = 0
	for link_id, link_uri, old in cursor:
		links_total += 1
		values = {}
		values['uri'] = link_uri
		# How old in seconds
		values['old'] = old
		values['w'] = 0
		values['c'] = 0
		values['v'] = 0
		values['links_order'] = links_total
		links[link_id] = values

	if not links_total:
		return

	links_format = ','.join(['%s'] * len(links))
	query = """
		select vote_link_id,
			sum((1-(unix_timestamp(now())
					- unix_timestamp(vote_date))/36000)) as x,
			count(*)
		from votes
		where vote_link_id in (%s)
			and vote_type='links'
			and vote_date > date_sub(now(), interval 24 hour)
			and vote_user_id > 0
			and vote_value >= 2
		group by vote_link_id
		order by x desc
	"""  % links_format
	cursor.execute(query, tuple(links))
	votes_total = 0
	votes_links = 0
	v_total = 0
	v_list = {}
	for link_id, old, votes in cursor:
		votes_links += 1
		votes_old = float(old)
		links[link_id]['v'] = votes_old
		v_total += votes_old
		v_list[link_id] = votes_old
		links[link_id]['votes'] = votes
		votes_total += votes
		links[link_id]['votes_order'] = votes_links

	if not votes_links:
		return

	v_average = v_total/votes_links
	votes_average = votes_total/votes_links

	query = """
		select comment_link_id,
			sum(1.5*(1-(unix_timestamp(now())
						- unix_timestamp(comment_date))/36000)),
			count(*)
		from comments
		where comment_link_id in (%s)
			and comment_date > date_sub(now(), interval 24 hour)
		group by comment_link_id
	""" % links_format
	cursor.execute(query, tuple(links))
	comments_total = 0
	comments_links = 0
	c_total = 0
	c_list = {}
	for link_id, old, count in cursor:
		comment_old = float(old)
		comments_links += 1
		links[link_id]['c'] = comment_old
		c_total += comment_old
		c_list[link_id] = comment_old
		links[link_id]['comments'] = count
		comments_total += count

	if not comments_links:
		return

	c_average = c_total/comments_links
	comments_average = comments_total/comments_links
	query = """
		select id, counter from link_clicks where id in (%s)
	""" % links_format

	cursor.execute(query, tuple(links))
	for link_id, clicks in cursor:
		links[link_id]['clicks'] = clicks

	cursor.close()

	print "*******************************************************************************************************"
	print "****", time.strftime("%d/%m/%Y, %H:%M:%S (%Z)"), "**** SITE:", site, ", Votes average:", votes_average, v_average, \
			", Comments average:", comments_average, c_average

	for link_id, link_value in links.items():
		if link_value['c'] > 0 \
				and link_value['v'] > 0 \
				and 'clicks' in link_value:
			links[link_id]['w'] = (1 - link_value['old']/(1.5*86400)) * (link_value['v'] \
						   + link_value['c'] \
						   + link_value['clicks'] * (1 - link_value['old']/86400) * 0.03)

{'fraction_non_alphanumeric': 0.16513505987190197, 'fraction_numerical': 0.016847674742411584, 'mean_word_length': 4.090715804394047, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6136787', 'n_tokens_mistral': 2906, 'n_tokens_neox': 2561, 'n_words': 638}