<commit_before># Copyright 2011, The Board of Regents of Leland Stanford, Jr. University
# All rights reserved. See LICENSE. 
# Author: <NAME> <<EMAIL>>
# Description: Dump events from calendar and tumblr feeds straight into database.

import sqlite3
import time
import urllib2

import gcal
import tumblr

events = gcal.parse_feed(urllib2.urlopen("https://www.google.com/calendar/feeds/stanford.humanities.events%40gmail.com/public/basic?showdeleted=true"))
news = tumblr.parse_rss(urllib2.urlopen("http://stanfordhumanitiescenter.tumblr.com/rss"))

def ToTimestamp(d):
	# STUPID PYTHON!!
	return int(time.mktime(d.timetuple()))

conn = sqlite3.connect("database.db")
c = conn.cursor()
for event in events:
	c.execute("insert into events values (?,?,?,?,?,?,?,?)", 
						(event.event_id, event.calendar_title, event.event_title,
						 ToTimestamp(event.start_time), ToTimestamp(event.end_time), event.location,
						 event.location, event.description))
for post in news:
	c.execute("insert into news values (?, ?, ?, ?, ?)",
						(post.post_id, post.post_title, ToTimestamp(post.post_date), post.post_content, 
						 post.post_category))
						
conn.commit()
c.close()<commit_msg>Refactor dumper to use the datastore API<commit_after># Copyright 2011, The Board of Regents of Leland Stanford, Jr. University
# All rights reserved. See LICENSE. 
# Author: <NAME> <<EMAIL>>
# Description: Dump events from calendar and tumblr feeds straight into database.


import urllib2

import datastore
import gcal
import tumblr

events = gcal.parse_feed(urllib2.urlopen("https://www.google.com/calendar/feeds/stanford.humanities.events%40gmail.com/public/basic?showdeleted=true"))
news = tumblr.parse_rss(urllib2.urlopen("http://stanfordhumanitiescenter.tumblr.com/rss"))

store = datastore.load("database.db")
store.update(events, news)
store.save()
store.close()
{'fraction_non_alphanumeric': 0.13254310344827586, 'fraction_numerical': 0.010775862068965518, 'mean_word_length': 5.294915254237288, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6011187', 'n_tokens_mistral': 657, 'n_tokens_neox': 599, 'n_words': 140}