<commit_before>import urllib
import urllib2
import time
import random
import datetime
from lxml import html


dayToInt = {'Lun': 1,
			'Mar': 2,
			'Mer': 3,
			'Jeu': 4,
			'Ven': 5,
			'Sam': 6,
			'Dim': 0}

def generateDatetime(year, weekNumber, day, hour):
	#day is Lun or Mar or...
	#year is YYYY
	#hour is : HH:mm
	d = str(year)+'-W'+str(weekNumber)+'-'+str(dayToInt[day])+'-'+hour
	#Ex: 2017-W43-5-14:00
	r = datetime.datetime.strptime(d, "%Y-W%W-%w-%H:%M")
	#iCal format:
	# 19980118T230000
	return r.strftime('%Y%m%dT%H%M%S')

def writeEvent(file, titre, debut, fin, description):
	file.write("BEGIN:VEVENT\n")
	file.write("DTSTART:"+debut+"\n")
	file.write("DTEND:"+fin+"\n")
	file.write("SUMMARY:"+titre+"\n")
	file.write("DESCRIPTION:"+description+"\n")
	file.write("END:VEVENT\n")

def writeCalendarHeader(file):
	file.write("BEGIN:VCALENDAR\n")
	file.write("VERSION:2.0\n")
	file.write("PRODID:Calendrier Metz\n")

def getDataFromEventNode(event):
	jour = event.xpath('./@id')[0][2:5]
	heure = event.xpath('.//td[@class="salle"]/text()')[0]
	try:
		heureDebut, heureFin = heure.split('-')
	except:
		print('Mauvais format heure :'+heure)
		sys.exit(1)

	coursListe = event.xpath('.//td[@class = "edtrv"]/text()')
	try:
		cours = coursListe[0]
	except:
		print("pas de cours trouve")
		pass

	try:
		cours += ' '+coursListe[1]
	except:
		pass

	try:
		prof = coursListe[2]
	except:
		prof = ''

	return jour, heureDebut, heureFin, cours, prof 

def getHTMLTreeForYearWeek(year,week, LOCAL):
	if LOCAL:
		print("Getting local HTML")
		with open('default.html', 'r') as f:
			content = f.read()
		f.close()
	else:
		print("Getting remote HTML for Y: "+str(year)+" W: "+str(week))

		opener = urllib2.build_opener()
		opener.addheaders = [('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'),
						 ('Authorization', 'Basic XXXXXXXXXX')]
		values = dict(Semaine=str(week), Annee=str(year), Sequence='3ASIR')
		data = urllib.urlencode(values)
		url = 'https://ntnoe.metz.supelec.fr/cgi-bin/agenda/AffEdTEleves.php?Semaine='+str(week)+'&Annee='+str(year)+'&Sequence=3ASIR&Concerne='
		response = opener.open(url, data)
		content = response.read()

	return html.fromstring(unicode(content, 'iso-8859-15'))

def main():

	F = open("calendar.ical", "w")
	writeCalendarHeader(F)

	year = 2018
	for week in range(1,14):

		tree = getHTMLTreeForYearWeek(year, week, LOCAL=False)

		for event in tree.xpath('//div[@id[starts-with(.,"RV")]]'):

			jour, heureDebut, heureFin, cours, prof = getDataFromEventNode(event)

			debut = generateDatetime(year, week, jour, heureDebut)
			fin = generateDatetime(year, week, jour, heureFin)
			
			print(cours)
			print(prof)
			print(heureDebut+" - "+heureFin)
			print('')

			writeEvent(F, cours.encode('utf-8'), debut, fin, prof.encode('utf-8'))


		time.sleep(random.randint(1, 5))

	F.write("END:VCALENDAR\n")
	F.close()


main()<commit_msg>Applied PEP8 recommandations for clarity<commit_after>import os
import time
import random
import datetime
import urllib
import urllib2

from lxml import html


dayToInt = {'Lun': 1, 'Mar': 2, 'Mer': 3, 'Jeu': 4, 'Ven': 5, 'Sam': 6, 'Dim': 0}


def generate_datetime(year, weekNumber, day, hour):
	# day is Lun or Mar or...
	# year is YYYY
	# hour is : HH:mm
	d = "{}-W{}-{}-{}".format(str(year), str(week_number), str(dayToInt[day]), hour)
	# Ex: 2017-W43-5-14:00
	r = datetime.datetime.strptime(d, "%Y-W%W-%w-%H:%M")
	# iCal format:
	# 19980118T230000
	return r.strftime('%Y%m%dT%H%M%S')


def write_event(file, titre, debut, fin, description):
	file.write("BEGIN:VEVENT\n")
	file.write("DTSTART:{}\n".format(debut))
	file.write("DTEND:{}\n".format(fin))
	file.write("SUMMARY:{}\n".format(titre))
	file.write("DESCRIPTION:{}\n".format(description))
	file.write("END:VEVENT\n")


def write_calendar_header(file):
	file.write("BEGIN:VCALENDAR\n")
	file.write("VERSION:2.0\n")
	file.write("PRODID:<NAME>\n")


def getDataFromEventNode(event):
	jour = event.xpath('./@id')[0][2:5]
	heure = event.xpath('.//td[@class="salle"]/text()')[0]
	try:
		heure_debut, heure_fin = heure.split('-')
	except:
		print('Mauvais format heure :{}'.format(heure))
		sys.exit(1)

	cours_liste = event.xpath('.//td[@class = "edtrv"]/text()')
	try:
		cours = cours_liste[0]
	except:
		print("pas de cours trouve")
		pass

	try:
		cours += ' ' + cours_liste[1]
	except:
		pass

	try:
		prof = cours_liste[2]
	except:
		prof = ''

	return jour, heure_debut, heure_fin, cours, prof


def getHTMLTreeForYearWeek(year, week, LOCAL):
	if LOCAL:
		print("Getting local HTML")
		with open('default.html', 'r') as f:
			content = f.read()
		f.close()
	else:
		print("Getting remote HTML for Y: {} W: {}".format(str(year), str(week)))

		opener = urllib2.build_opener()
		opener.addheaders = [('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'),
						 ('Authorization', 'Basic XXXXXXXXXX')]
		values = dict(Semaine=str(week), Annee=str(year), Sequence='3ASIR')
		data = urllib.urlencode(values)
		url = 'https://ntnoe.metz.supelec.fr/cgi-bin/agenda/AffEdTEleves.php?Semaine='+str(week)+'&Annee='+str(year)+'&Sequence=3ASIR&Concerne='
		response = opener.open(url, data)
		content = response.read()

	return html.fromstring(unicode(content, 'iso-8859-15'))


def main():

	calendar_file = open("calendar.ical", "w")
	write_calendar_header(calendar_file)

	year = 2018
	for week in range(1,14):

		tree = getHTMLTreeForYearWeek(year, week, LOCAL=False)

		for event in tree.xpath('//div[@id[starts-with(.,"RV")]]'):

			jour, heure_debut, heure_fin, cours, prof = getDataFromEventNode(event)

			debut = generate_datetime(year, week, jour, heure_debut)
			fin = generate_datetime(year, week, jour, heure_fin)

			print(cours)
			print(prof)
			print("{} - {}".format(heure_debut, heure_fin))
			print('')

			write_event(calendar_file, cours.encode('utf-8'), debut, fin, prof.encode('utf-8'))

		time.sleep(random.randint(1, 5))

	calendar_file.write("END:VCALENDAR\n")
	calendar_file.close()


if __name__ == '__main__':
	main()
{'fraction_non_alphanumeric': 0.18999675219227022, 'fraction_numerical': 0.029392659954530692, 'mean_word_length': 4.723977695167286, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1573426', 'n_tokens_mistral': 2767, 'n_tokens_neox': 2488, 'n_words': 476}