<commit_before># Scraper for Pomona (Frank, Frary) dining halls.

from codecs import open
from datetime import date
from datetime import timedelta
import json
import lxml.html
import os
import re
import requests
import datetime
from urllib import quote

PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))

# Oldenborg scrapping is a little different
is_oldenborg = False

class GoogleWorksheet(object):
	def __init__(self, w):
		self.w = w

	def __repr__(self):
		return repr(self.w)

	def __getitem__(self, key):
		return self.w[key]

	def _parse_hyphenated_date(self, s):
		try:
			month, day, year = map(int, s.split("-"))
			return date(2000 + year, month, day)
		except ValueError:
			return datetime.datetime.today()

	def date(self):
		return self._parse_hyphenated_date(self.w['title']['$t'])

class GoogleCell(object):
	def __init__(self, c):
		position = c['title']['$t']
		self.column = str(position[0])
		self.row = int(position[1:])
		value = c['content']['$t']
		value = re.sub(u'[\xa0\s]+', ' ', value)

		if is_oldenborg:
			self.value = value.strip().encode('utf-8') # Oldenborg cells contain one fooditem each
		else:
			self.value = value.strip().encode('utf-8').split(',') # Creates a list

	def __repr__(self):
		return "(%s, %i, %s)" % (self.column, self.row, self.value)

class NotFoundException(Exception): pass

class PomonaBackend(object):
	def _I(self, x): return x

	def _download(self, (url, after)):
		return after(self._download_helper_maybe_cache(url))

	def _download_helper_maybe_cache(self, url):
		# if debugging, comment out this line to go faster!
		return requests.get(url).text

		cachename = os.path.join(
	        PROJECT_ROOT,
	        "cache",
	        "{0}.cache".format(quote(url, safe=''))
	    )
		try:
			with open(cachename) as f:
				return f.read()
		except:
			data = requests.get(url).text
			with open(cachename, "w", encoding="utf-8") as f:
				f.write(data)
			return data

	def _find_spreadsheet_id(self, data):
		doc = lxml.html.fromstring(data)
		spreadsheet_id = doc.xpath('//*[@data-google-spreadsheet-id]')
		assert len(spreadsheet_id) == 1
		return spreadsheet_id[0].get('data-google-spreadsheet-id')

	def _get_worksheets_for_key(self, key):
		def after(resp):
			worksheets_json = json.loads(resp)
			worksheets = worksheets_json['feed']['entry']
			return map(GoogleWorksheet, worksheets)

		worksheets_url_template = "https://spreadsheets.google.com/feeds/worksheets/{key}/public/basic?alt=json"
		worksheets_url = worksheets_url_template.format(key=key)

		return (worksheets_url, after)

	def _get_cells_feed(self, worksheet):
		def after(resp):
			cells = json.loads(resp)['feed']['entry']
			return map(GoogleCell, cells)

		links = worksheet['link']
		rel = "http://schemas.google.com/spreadsheets/2006#cellsfeed"
		cells_links = filter(lambda l: l['rel'] == rel, links)
		assert len(cells_links) == 1
		cells_url = cells_links[0]['href'] + "?alt=json"

		return (cells_url, after)

	def _most_recent_monday(self, d):
		return d - timedelta(days=d.weekday())

	def _find(self, cond, seq):
		try:
			return next(iter(filter(cond, seq)))
		except StopIteration:
			raise NotFoundException(cond, seq)

	def _parse_frank_frary_cells(self, cells):
		#  Spreadsheet format
		#
		#    |   A  |   B       |    C    |  D     |  E     |
		#  1 |      |           |         |        |        |
		#  2 |  Day |Station    |Breakfast|Lunch   |Dinner  |
		#  3 |Monday|Mealname   |         |        |        |<commit_msg>Refactors PomonaBackend to change class hierarchy.<commit_after># Scraper for Pomona (Frank, Frary) dining halls.

from codecs import open
from datetime import date
from datetime import timedelta
import json
import lxml.html
import os
import re
import requests
import datetime
from urllib import quote

PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))

# Oldenborg scrapping is a little different
is_oldenborg = False

class PomonaBackend(object):
	class GoogleWorksheet(object):
		def __init__(self, w):
			self.w = w

		def __repr__(self):
			return repr(self.w)

		def __getitem__(self, key):
			return self.w[key]

		def _parse_hyphenated_date(self, s):
			try:
				month, day, year = map(int, s.split("-"))
				return date(2000 + year, month, day)
			except ValueError:
				return datetime.datetime.today()

		def date(self):
			return self._parse_hyphenated_date(self.w['title']['$t'])

	class GoogleCell(object):
		def __init__(self, c):
			position = c['title']['$t']
			self.column = str(position[0])
			self.row = int(position[1:])
			value = c['content']['$t']
			value = re.sub(u'[\xa0\s]+', ' ', value)

			if is_oldenborg:
				self.value = value.strip().encode('utf-8') # Oldenborg cells contain one fooditem each
			else:
				self.value = value.strip().encode('utf-8').split(',') # Creates a list

		def __repr__(self):
			return "(%s, %i, %s)" % (self.column, self.row, self.value)

	class NotFoundException(Exception):
		pass

	def _I(self, x): return x

	def _download(self, (url, after)):
		return after(self._download_helper_maybe_cache(url))

	def _download_helper_maybe_cache(self, url):
		# if debugging, comment out this line to go faster!
		return requests.get(url).text

		cachename = os.path.join(
	        PROJECT_ROOT,
	        "cache",
	        "{0}.cache".format(quote(url, safe=''))
	    )
		try:
			with open(cachename) as f:
				return f.read()
		except:
			data = requests.get(url).text
			with open(cachename, "w", encoding="utf-8") as f:
				f.write(data)
			return data

	def _find_spreadsheet_id(self, data):
		doc = lxml.html.fromstring(data)
		spreadsheet_id = doc.xpath('//*[@data-google-spreadsheet-id]')
		assert len(spreadsheet_id) == 1
		return spreadsheet_id[0].get('data-google-spreadsheet-id')

	def _get_worksheets_for_key(self, key):
		def after(resp):
			worksheets_json = json.loads(resp)
			worksheets = worksheets_json['feed']['entry']
			return map(self.GoogleWorksheet, worksheets)

		worksheets_url_template = "https://spreadsheets.google.com/feeds/worksheets/{key}/public/basic?alt=json"
		worksheets_url = worksheets_url_template.format(key=key)

		return (worksheets_url, after)

	def _get_cells_feed(self, worksheet):
		def after(resp):
			cells = json.loads(resp)['feed']['entry']
			return map(self.GoogleCell, cells)

		links = worksheet['link']
		rel = "http://schemas.google.com/spreadsheets/2006#cellsfeed"
		cells_links = filter(lambda l: l['rel'] == rel, links)
		assert len(cells_links) == 1
		cells_url = cells_links[0]['href'] + "?alt=json"

		return (cells_url, after)

	def _most_recent_monday(self, d):
		return d - timedelta(days=d.weekday())

	def _find(self, cond, seq):
		try:
			return next(iter(filter(cond, seq)))
		except StopIteration:
			raise self.NotFoundException(cond, seq)

	def _parse_frank_frary_cells(self, cells):
		#  Spreadsheet format
		#
		#    |   A  |   B       |    C    |  D     |  E     |
		#  1 |      |           |         |        |        |
		#  2 |  Day |Station    |Breakfast|Lunch   |Dinner  |
		#  3 |Monday|Mealname   |         |        |        |
{'fraction_non_alphanumeric': 0.13975903614457832, 'fraction_numerical': 0.006236711552090716, 'mean_word_length': 3.675944333996024, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1977703', 'n_tokens_mistral': 2779, 'n_tokens_neox': 2461, 'n_words': 559}