<commit_before>from django.db import models
import lxml.etree
import datetime
import time
import sys
import urllib2
import foojson
import json

URL_CARTO = 'http://www.bicikelj.si/service/carto'
URL_STATION = 'http://www.bicikelj.si/service/stationdetails/ljubljana/%s'

class BicikeljData(models.Model):
	timestamp = models.DateTimeField(auto_now_add=True, db_index=True)
	json_data = models.TextField()

def _parse_main_xml(data):
	json = {}
	carto = lxml.etree.fromstring(data)
	
	count = 0
	for marker in carto.xpath('//carto/markers/marker'):
		assert len(marker.getchildren()) == 0, 'Marker has children!? ' + repr(marker.getchildren())
		station_id = unicode(marker.attrib['number'])
		json[station_id] = dict(marker.attrib)
	
		count += 1
	assert count > 0, 'No markers found in main XML!?'
	
	return json

def _parse_station_xml(data):
	sta_xml = lxml.etree.fromstring(data)
	sta_dict = {}
	for elem in sta_xml.xpath('//station/*'):
		sta_dict[elem.tag] = elem.text
	assert list(sorted(sta_dict.keys())) == ['available', 'connected', 'free', 'open', 'ticket', 'total', 'updated'], 'Missing or added keys in station XML!?'
	# backwards compatibility
	del sta_dict['connected']
	del sta_dict['open']
	del sta_dict['updated']
	return sta_dict

def fetch_xmls():
	main_xml = urllib2.urlopen(URL_CARTO, timeout=10).read()
	json = _parse_main_xml(main_xml)
	
	for k, v in json.iteritems():
		sta_xml_data = urllib2.urlopen(URL_STATION % k, timeout=10).read()
		sta_dict = _parse_station_xml(sta_xml_data)
		v['station'] = sta_dict
		v['station_valid'] = not int(sta_dict['total']) == 0
	
	now = datetime.datetime.now()
	resp = {
		'markers': json,
		'updated': time.mktime(now.timetuple())
		}
	return now, resp

def convert_citybikes(data):
	try:
		citybikes = json.loads(data)
	except ValueError, e:
		print >> sys.stderr, [data]
		raise
		
	markers = {}
	updateds = []
	
	for d in citybikes:
		st_id = d['id']
		u = datetime.datetime.strptime(d['timestamp'], '%Y-%m-%dT%H:%M:%S.%f')
		updated = int(time.mktime(u.timetuple()))
		updateds.append(updated)
		lat = str(d['lat'])
		lng = str(d['lng'])
		markers[st_id] = {
			u'name': d['name'],
			u'fullAddress': d['name'],<commit_msg>Quick fix for the datetime parsing.<commit_after>from django.db import models
import lxml.etree
import datetime
import time
import sys
import urllib2
import foojson
import json

URL_CARTO = 'http://www.bicikelj.si/service/carto'
URL_STATION = 'http://www.bicikelj.si/service/stationdetails/ljubljana/%s'

class BicikeljData(models.Model):
	timestamp = models.DateTimeField(auto_now_add=True, db_index=True)
	json_data = models.TextField()

def _parse_main_xml(data):
	json = {}
	carto = lxml.etree.fromstring(data)
	
	count = 0
	for marker in carto.xpath('//carto/markers/marker'):
		assert len(marker.getchildren()) == 0, 'Marker has children!? ' + repr(marker.getchildren())
		station_id = unicode(marker.attrib['number'])
		json[station_id] = dict(marker.attrib)
	
		count += 1
	assert count > 0, 'No markers found in main XML!?'
	
	return json

def _parse_station_xml(data):
	sta_xml = lxml.etree.fromstring(data)
	sta_dict = {}
	for elem in sta_xml.xpath('//station/*'):
		sta_dict[elem.tag] = elem.text
	assert list(sorted(sta_dict.keys())) == ['available', 'connected', 'free', 'open', 'ticket', 'total', 'updated'], 'Missing or added keys in station XML!?'
	# backwards compatibility
	del sta_dict['connected']
	del sta_dict['open']
	del sta_dict['updated']
	return sta_dict

def fetch_xmls():
	main_xml = urllib2.urlopen(URL_CARTO, timeout=10).read()
	json = _parse_main_xml(main_xml)
	
	for k, v in json.iteritems():
		sta_xml_data = urllib2.urlopen(URL_STATION % k, timeout=10).read()
		sta_dict = _parse_station_xml(sta_xml_data)
		v['station'] = sta_dict
		v['station_valid'] = not int(sta_dict['total']) == 0
	
	now = datetime.datetime.now()
	resp = {
		'markers': json,
		'updated': time.mktime(now.timetuple())
		}
	return now, resp

def convert_citybikes(data):
	try:
		citybikes = json.loads(data)
	except ValueError, e:
		print >> sys.stderr, [data]
		raise
		
	markers = {}
	updateds = []
	
	for d in citybikes:
		st_id = d['id']
		u = datetime.datetime.strptime(d['timestamp'].rstrip('Z'), '%Y-%m-%dT%H:%M:%S.%f')
		updated = int(time.mktime(u.timetuple()))
		updateds.append(updated)
		lat = str(d['lat'])
		lng = str(d['lng'])
		markers[st_id] = {
			u'name': d['name'],
			u'fullAddress': d['name'],
{'fraction_non_alphanumeric': 0.16047144152311876, 'fraction_numerical': 0.005439709882139619, 'mean_word_length': 4.868351063829787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 28}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2116334', 'n_tokens_mistral': 1746, 'n_tokens_neox': 1642, 'n_words': 344}