<commit_before>import re, asyncore, feedparser
from time import sleep
from ConfigParser import ConfigParser
from sys import argv, exit
from ircasync import *
from subprocess import Popen, PIPE
from thread import start_new_thread

config = ConfigParser()
try:
	config.readfp(open(argv[1]))
except:
	try:
		config.readfp(open('feedbot.ini'))
	except:
		print "Syntax:"
		print "  %s [config]" % argv[0]
		print ""
		print "If no configuration file is specified or there was an error, it will default to `feedbot.ini'."
		print "If there was a failure reading the configuration, it will display this message."
		exit(1)

# read config
SERVER = config.get('feedbot', 'server')
try: PORT = config.getint('feedbot', 'port')
except: PORT = DEFAULT_PORT
NICK = config.get('feedbot', 'nick')
CHANNEL = config.get('feedbot', 'channel')
VERSION = 'feedbot hg:%s; http://hg.micolous.id.au/ircbots/'
try: VERSION = VERSION % Popen(["hg","id"], stdout=PIPE).communicate()[0].strip()
except: VERSION = VERSION % 'unknown'
del Popen, PIPE

try: NICKSERV_PASS = config.get('feedbot', 'nickserv_pass')
except: NICKSERV_PASS = None
try: UPDATE_FREQUENCY = config.getint('feedbot', 'update_frequency')
except: UPDATE_FREQUENCY = 300

feed_urls = {}
if config.has_section('feeds'):
	for k,v in config.items('feeds'):
		feed_urls[k] = v

feeds = {}
last_feeds = {}


def announce_post(irc, feed, entry):
	global CHANNEL
	print 'NEW POST: %s: %s (%s)' % (entry.title, entry.link, feed)
	irc.action(CHANNEL, 'found a new post on %s: %s: %s' % (str(feed), str(entry.title.strip()), str(entry.link.strip())))

def update(irc):
	global feed_urls, feeds, last_feeds
	
	last_feeds = feeds
	feeds = {}
	
	# work though the urls
	for k in feed_urls:
		try:
			# download the new feed
			if last_feeds.has_key(k) and hasattr(last_feeds[k], 'modified'):
				feeds[k] = feedparser.parse(feed_urls[k], modified=last_feeds[k].modified)<commit_msg>redirect all forum posts through fj.streetgeek.* for shorter urls<commit_after>import re, asyncore, feedparser
from time import sleep
from ConfigParser import ConfigParser
from sys import argv, exit
from ircasync import *
from subprocess import Popen, PIPE
from thread import start_new_thread

config = ConfigParser()
try:
	config.readfp(open(argv[1]))
except:
	try:
		config.readfp(open('feedbot.ini'))
	except:
		print "Syntax:"
		print "  %s [config]" % argv[0]
		print ""
		print "If no configuration file is specified or there was an error, it will default to `feedbot.ini'."
		print "If there was a failure reading the configuration, it will display this message."
		exit(1)

# read config
SERVER = config.get('feedbot', 'server')
try: PORT = config.getint('feedbot', 'port')
except: PORT = DEFAULT_PORT
NICK = config.get('feedbot', 'nick')
CHANNEL = config.get('feedbot', 'channel')
VERSION = 'feedbot hg:%s; http://hg.micolous.id.au/ircbots/'
try: VERSION = VERSION % Popen(["hg","id"], stdout=PIPE).communicate()[0].strip()
except: VERSION = VERSION % 'unknown'
del Popen, PIPE

try: NICKSERV_PASS = config.get('feedbot', 'nickserv_pass')
except: NICKSERV_PASS = None
try: UPDATE_FREQUENCY = config.getint('feedbot', 'update_frequency')
except: UPDATE_FREQUENCY = 300

feed_urls = {}
if config.has_section('feeds'):
	for k,v in config.items('feeds'):
		feed_urls[k] = v

feeds = {}
last_feeds = {}


def announce_post(irc, feed, entry):
	global CHANNEL
	print 'NEW POST: %s: %s (%s)' % (entry.title, entry.link.replace("http://www.streetgeek.com.au/portal/modules.php?name=Forums&file=viewtopic&p=", "http://fj.streetgeek.com.au/"), feed)
	irc.action(CHANNEL, 'found a new post on %s: %s: %s' % (str(feed), str(entry.title.strip()), str(entry.link.strip().replace("http://www.streetgeek.com.au/portal/modules.php?name=Forums&file=viewtopic&p=", "http://fj.streetgeek.com.au/"))))

def update(irc):
	global feed_urls, feeds, last_feeds
	
	last_feeds = feeds
	feeds = {}
	
	# work though the urls
	for k in feed_urls:
		try:
			# download the new feed
			if last_feeds.has_key(k) and hasattr(last_feeds[k], 'modified'):
				feeds[k] = feedparser.parse(feed_urls[k], modified=last_feeds[k].modified)
{'fraction_non_alphanumeric': 0.1489310009718173, 'fraction_numerical': 0.003401360544217687, 'mean_word_length': 5.172413793103448, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '699962', 'n_tokens_mistral': 1529, 'n_tokens_neox': 1468, 'n_words': 393}