<commit_before>#!/usr/bin/env python3
import string, os, time, json, re, unicodedata, html.parser, requests

# Bon appetit cafe hours api url
url = "http://legacy.cafebonappetit.com/api/2/cafes"
# How many cafeterias you want to parse (in order)
totalCafes = 1818
# What our file should be named
fileName = "data.json"


# Parser to clean up the string names
h = html.parser.HTMLParser()
# Time when we started the script
start = time.time()


def clean(stringToClean):
	# Remove beginning and ending whitespace
	string = stringToClean.strip()
	# Replace html
	cleanString = h.unescape(string)
	# Replace unicode
	return unicodedata.normalize('NFKD', cleanString)


# Finds the cafeteria id and name
def getBonAppMenuData(url, id):
	response = requests.get(url, params={'cafe': id})
	data = response.json()

	try:
		# Grab cafe Id from JSON
		cafeId = list(data["cafes"].keys())[0]

		# Grab cafe name from JSON
		cafeName = data["cafes"][cafeId]["name"]


		# We want to display titles at least. We can do without location.
		# Grab cafe city from JSON
		cafeCity = data["cafes"][cafeId].get("city", "")

		# Grab cafe state from JSON
		cafeState = data["cafes"][cafeId].get("state", "")

		# Formatting city and state strings
		# Both empty
		if cafeCity == "" and cafeState == "":
			cafeLoc = "No location listed"
		# Only city
		elif cafeCity != "" and cafeState == "":
			cafeLoc = clean(cafeCity)
		# Only State
		elif cafeCity == "" and cafeState != "":
			cafeLoc = clean(cafeState)
		# City and State
		else:
			cafeLoc = clean(cafeCity) + ", " + clean(cafeState)


		# Clean up the cafe name
		cafeName = clean(cafeName)

		# Construct the full return string
		print(cafeId + ") " + cafeName + " in " + cafeLoc)
		return {'id':cafeId, 'label':cafeName, 'desc':cafeLoc}
	except:
		print('[Skipping. Moving on...]')
		pass

# Round numbers to a decimal point
def num2str(num, precision):
	return "%0.*f" % (precision, num)

# Get the outfile's size
def calculateFileSize():
	return str(os.path.getsize(fileName))


def main():
	# Our constructed JSON data
	responseData = []

	# Loop through the "known" amount of cafes
	for num in range(0, totalCafes):
		data = getBonAppMenuData(url, num)
		# Append if it's not null
		if data is not None:
			responseData.append(data)
		time.sleep(3)

	# Write our output to a file
	with open(fileName, 'w') as outfile:
		# Output the data into a file
		json.dump(responseData, outfile)
		# Save the runtime
		endTime = time.time() - start;

	print('')
	print('File: ' + fileName)
	print('Size: ' + calculateFileSize() + ' bytes')
	print('This took ' + num2str(endTime, 2) + ' seconds\n')

if __name__ == '__main__':
	main()
<commit_msg>Bump total number of cafes<commit_after>#!/usr/bin/env python3
import string, os, time, json, re, unicodedata, html.parser, requests

# Bon appetit cafe hours api url
url = "http://legacy.cafebonappetit.com/api/2/cafes"
# How many cafeterias you want to parse (in order)
totalCafes = 2393
# What our file should be named
fileName = "data.json"


# Parser to clean up the string names
h = html.parser.HTMLParser()
# Time when we started the script
start = time.time()


def clean(stringToClean):
	# Remove beginning and ending whitespace
	string = stringToClean.strip()
	# Replace html
	cleanString = h.unescape(string)
	# Replace unicode
	return unicodedata.normalize('NFKD', cleanString)


# Finds the cafeteria id and name
def getBonAppMenuData(url, id):
	response = requests.get(url, params={'cafe': id})
	data = response.json()

	try:
		# Grab cafe Id from JSON
		cafeId = list(data["cafes"].keys())[0]

		# Grab cafe name from JSON
		cafeName = data["cafes"][cafeId]["name"]


		# We want to display titles at least. We can do without location.
		# Grab cafe city from JSON
		cafeCity = data["cafes"][cafeId].get("city", "")

		# Grab cafe state from JSON
		cafeState = data["cafes"][cafeId].get("state", "")

		# Formatting city and state strings
		# Both empty
		if cafeCity == "" and cafeState == "":
			cafeLoc = "No location listed"
		# Only city
		elif cafeCity != "" and cafeState == "":
			cafeLoc = clean(cafeCity)
		# Only State
		elif cafeCity == "" and cafeState != "":
			cafeLoc = clean(cafeState)
		# City and State
		else:
			cafeLoc = clean(cafeCity) + ", " + clean(cafeState)


		# Clean up the cafe name
		cafeName = clean(cafeName)

		# Construct the full return string
		print(cafeId + ") " + cafeName + " in " + cafeLoc)
		return {'id':cafeId, 'label':cafeName, 'desc':cafeLoc}
	except:
		print('[Skipping. Moving on...]')
		pass

# Round numbers to a decimal point
def num2str(num, precision):
	return "%0.*f" % (precision, num)

# Get the outfile's size
def calculateFileSize():
	return str(os.path.getsize(fileName))


def main():
	# Our constructed JSON data
	responseData = []

	# Loop through the "known" amount of cafes
	for num in range(1, totalCafes):
		data = getBonAppMenuData(url, num)
		# Append if it's not null
		if data is not None:
			responseData.append(data)
		time.sleep(3)

	# Write our output to a file
	with open(fileName, 'w') as outfile:
		# Output the data into a file
		json.dump(responseData, outfile)
		# Save the runtime
		endTime = time.time() - start;

	print('')
	print('File: ' + fileName)
	print('Size: ' + calculateFileSize() + ' bytes')
	print('This took ' + num2str(endTime, 2) + ' seconds\n')

if __name__ == '__main__':
	main()

{'fraction_non_alphanumeric': 0.13092802678073276, 'fraction_numerical': 0.004835410079970244, 'mean_word_length': 3.92942254812099, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1157313', 'n_tokens_mistral': 2009, 'n_tokens_neox': 1850, 'n_words': 578}