<commit_before>#!/usr/bin/env python

import urllib2
from HTMLParser import HTMLParser
import datetime
import re
import xml.etree.ElementTree as et

class UnipiParser(HTMLParser):
	def __init__(self):
		HTMLParser.__init__(self)
		
		self.innorm = False
		self.ina = False
		self.infont = False
		self.table_depth = 0
		self.font_depth = 0
		
		self.articles = []
	
	def handle_starttag(self, tag, attrs):
		if tag == 'table':
			if self.innorm:
				self.table_depth += 1
			else:
				for attr in attrs:
					if attr[0] == 'class' and attr[1] == 'norm':
						self.table_depth = 1
						self.innorm = True

		if tag == 'font' and self.innorm:
			self.infont = True
			self.font_depth += 1

		if tag == 'a' and self.infont:
			self.ina = True
			for attr in attrs:
				if attr[0] == 'href':
					self.articles.append(attr[1])
			
	
	def handle_endtag(self, tag):
		if tag == 'table':
			self.table_depth -=1
			if self.table_depth == 0:
				self.innorm = False

		if tag == 'font':
			self.font_depth -= 1
			if self.font_depth == 0:
				self.infont = False

		if tag == 'a':
			self.ina = False

class Stripper(HTMLParser):
	def __init__(self):
		HTMLParser.__init__(self)
		self.ob = ''
	def handle_data(self, data):
		self.ob += data
        
def strip(html):
    s = Stripper()
    s.feed(html)
    return s.ob

html = urllib2.urlopen('http://www.unipi.gr').read()
parser = UnipiParser()
parser.feed(html)

root = et.Element('feed')
root.set('xmlns', 'http://www.w3.org/2005/Atom')

global_title_node = et.SubElement(root, 'title')
global_title_node.text = 'Unipi feed'
icon_node = et.SubElement(root, 'icon')
icon_node.text = 'http://www.unipi.gr/favicon.ico'
global_id_node = et.SubElement(root, 'id')
global_id_node.text = 'urn:students.cs.unipi.gr-feed'
global_updated_node = et.SubElement(root, 'updated')
global_updated_node.text = datetime.datetime.now().strftime('%Y-%m-%dT%H-%M-%SZ')

for url in parser.articles:
	try:
		html = urllib2.urlopen('http://www.unipi.gr/%s' % url).read()
	except urllib2.HTTPError, e:
		pass
	else:
		html = html[html.find('smaplev2') + 5:]
		title = re.findall('<p class="smaplev2">(.*?)</p>', html, re.S)[0].decode('cp1253')
		title = strip(title)
		datestr = re.findall('<p class="smaplev2">.*?</p>.*?<p>.*?<strong>(.*?)</strong>.*?</p>', html, re.S)[0].strip().decode('cp1253')
		date = datetime.datetime.strptime(datestr, '%d/%m/%Y')
		content = re.findall('<td class="norm"><p class="smaplev2">.*?</p>.*?<p>.*?</p>(.*?)</td>', html, re.S)[0].strip().decode('cp1253')
		id = url.split('=')[-1]
	
		entry_node = et.SubElement(root, 'entry')
		title_node = et.SubElement(entry_node, 'title', type="html")
		title_node.text = title
		link_node = et.SubElement(entry_node, 'link')
		link_node.set('href', 'http://www.unipi.gr/%s' % url)
		id_node = et.SubElement(entry_node, 'id')
		id_node.text = 'urn:article-%s' % id
		updated_node = et.SubElement(entry_node, 'updated')
		updated_node.text = date.strftime('%Y-%m-%dT00-00-00Z')
		summary_node = et.SubElement(entry_node, 'content', mode="escaped", type="text/html")
		summary_node.text = content
	
tree = et.ElementTree(root)
tree.write('miou.atom', encoding="utf-8")<commit_msg>Just an ugly regexp hack for absolute urls.<commit_after>#!/usr/bin/env python

import urllib2
from HTMLParser import HTMLParser
import datetime
import re
import xml.etree.ElementTree as et

class UnipiParser(HTMLParser):
	def __init__(self):
		HTMLParser.__init__(self)
		
		self.innorm = False
		self.ina = False
		self.infont = False
		self.table_depth = 0
		self.font_depth = 0
		
		self.articles = []
	
	def handle_starttag(self, tag, attrs):
		if tag == 'table':
			if self.innorm:
				self.table_depth += 1
			else:
				for attr in attrs:
					if attr[0] == 'class' and attr[1] == 'norm':
						self.table_depth = 1
						self.innorm = True

		if tag == 'font' and self.innorm:
			self.infont = True
			self.font_depth += 1

		if tag == 'a' and self.infont:
			self.ina = True
			for attr in attrs:
				if attr[0] == 'href':
					self.articles.append(attr[1])
			
	
	def handle_endtag(self, tag):
		if tag == 'table':
			self.table_depth -=1
			if self.table_depth == 0:
				self.innorm = False

		if tag == 'font':
			self.font_depth -= 1
			if self.font_depth == 0:
				self.infont = False

		if tag == 'a':
			self.ina = False

class Stripper(HTMLParser):
	def __init__(self):
		HTMLParser.__init__(self)
		self.ob = ''
	def handle_data(self, data):
		self.ob += data
        
def strip(html):
    s = Stripper()
    s.feed(html)
    return s.ob

html = urllib2.urlopen('http://www.unipi.gr').read()
parser = UnipiParser()
parser.feed(html)

root = et.Element('feed')
root.set('xmlns', 'http://www.w3.org/2005/Atom')

global_title_node = et.SubElement(root, 'title')
global_title_node.text = 'Unipi feed'
icon_node = et.SubElement(root, 'icon')
icon_node.text = 'http://www.unipi.gr/favicon.ico'
global_id_node = et.SubElement(root, 'id')
global_id_node.text = 'urn:students.cs.unipi.gr-feed'
global_updated_node = et.SubElement(root, 'updated')
global_updated_node.text = datetime.datetime.now().strftime('%Y-%m-%dT%H-%M-%SZ')

for url in parser.articles:
	try:
		html = urllib2.urlopen('http://www.unipi.gr/%s' % url).read()
	except urllib2.HTTPError, e:
		pass
	else:
		html = html[html.find('smaplev2') + 5:]
		title = re.findall('<p class="smaplev2">(.*?)</p>', html, re.S)[0].decode('cp1253')
		title = strip(title)
		datestr = re.findall('<p class="smaplev2">.*?</p>.*?<p>.*?<strong>(.*?)</strong>.*?</p>', html, re.S)[0].strip().decode('cp1253')
		date = datetime.datetime.strptime(datestr, '%d/%m/%Y')
		content_a = re.findall('<td class="norm"><p class="smaplev2">.*?</p>.*?<p>.*?</p>(.*?)</td>', html, re.S)[0].strip().decode('cp1253')
		content = re.sub(r"(?<=href=['\"])((?!http).*?)(?=['\"])", 'http://www.unipi.gr/'+r'\1', content_a)
		id = url.split('=')[-1]

		entry_node = et.SubElement(root, 'entry')
		title_node = et.SubElement(entry_node, 'title', type="html")
		title_node.text = title
		link_node = et.SubElement(entry_node, 'link')
		link_node.set('href', 'http://www.unipi.gr/%s' % url)
		id_node = et.SubElement(entry_node, 'id')
		id_node.text = 'urn:article-%s' % id
		updated_node = et.SubElement(entry_node, 'updated')
		updated_node.text = date.strftime('%Y-%m-%dT00-00-00Z')
		summary_node = et.SubElement(entry_node, 'content', mode="escaped", type="text/html")
		summary_node.text = content
	
tree = et.ElementTree(root)
tree.write('miou.atom', encoding="utf-8")
{'fraction_non_alphanumeric': 0.19040271563030398, 'fraction_numerical': 0.015584014812528931, 'mean_word_length': 3.877351392024078, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 11, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6216920', 'n_tokens_mistral': 2789, 'n_tokens_neox': 2521, 'n_words': 478}