<commit_before>#!/usr/bin/env python3
import string, os, time, json, re, unicodedata, html.parser, requests

# Bon appetit cafe hours api url
url = "http://legacy.cafebonappetit.com/api/2/cafes"
# How many cafeterias you want to parse (in order)
totalCafes = 1818
# What our file should be named
fileName = "data.json"


# Our constructed JSON data
responseData = []
# Parser to clean up the string names
h = html.parser.HTMLParser()
# Time when we started the script
start = time.time()

def appendData(cafeId, cafeName, cafeLoc):
	print(cafeId + ") " + cafeName + " in " + cafeLoc)
	responseData.append({'id':cafeId, 'label':cafeName, 'desc':cafeLoc})


def clean(stringToClean):
	# Remove beginning and ending whitespace
	string = stringToClean.strip()
	# Replace html
	cleanString = h.unescape(string)
	# Replace unicode
	cleanString2 = unicodedata.normalize('NFKD', cleanString)
	return cleanString2


# Finds the cafeteria id and name
def getBonAppMenuData(url, id):
	params = {'cafe': id}
	response = requests.get(url, params=params)
	data = response.text

	try:
		# Grab cafe Id from JSON
		cafeId = list(json.loads(data)["cafes"].keys())[0]

		# Grab cafe name from JSON
		cafeName = json.loads(data)["cafes"][cafeId]["name"]


		# We want to display titles at least. We can do without location.
		try:
			# Grab cafe city from JSON
			cafeCity = json.loads(data)["cafes"][cafeId]["city"]
		except:
			cafeCity = ""
		try:
			# Grab cafe state from JSON
			cafeState = json.loads(data)["cafes"][cafeId]["state"]
		except:
			cafeState = ""


		# Formatting city and state strings
		# Both empty
		if cafeCity == "" and cafeState == "":
			cafeLoc = "No location listed"
		# Only city
		elif cafeCity != "" and cafeState == "":
			cafeLoc = clean(cafeCity)
		# Only State
		elif cafeCity == "" and cafeState != "":
			cafeLoc = clean(cafeState)
		# City and State
		else:
			cafeLoc = clean(cafeCity) + ", " + clean(cafeState)


		# Clean up the cafe name
		cafeName = clean(cafeName)

		# Construct the full return string
		appendData(cafeId, cafeName, cafeLoc)
	except:
		print('[Skipping. Moving on...]')
		pass

# Round numbers to a decimal point
def num2str(num, precision):
	return "%0.*f" % (precision, num)

# Get the outfile's size
def calculateFileSize():
	fileSize = os.path.getsize(fileName)
	fileSize = str(fileSize)
	return fileSize


# Loop through the "known" amount of cafes
for num in range(0, totalCafes):
	getBonAppMenuData(url, num)
	time.sleep(3)

# Write our output to a file
with open(fileName, 'w') as outfile:
	# Output the data into a file
	json.dump(responseData, outfile)
	# Save the runtime
	endTime = time.time() - start;<commit_msg>Condense function to single line<commit_after>#!/usr/bin/env python3
import string, os, time, json, re, unicodedata, html.parser, requests

# Bon appetit cafe hours api url
url = "http://legacy.cafebonappetit.com/api/2/cafes"
# How many cafeterias you want to parse (in order)
totalCafes = 1818
# What our file should be named
fileName = "data.json"


# Our constructed JSON data
responseData = []
# Parser to clean up the string names
h = html.parser.HTMLParser()
# Time when we started the script
start = time.time()

def appendData(cafeId, cafeName, cafeLoc):
	print(cafeId + ") " + cafeName + " in " + cafeLoc)
	responseData.append({'id':cafeId, 'label':cafeName, 'desc':cafeLoc})


def clean(stringToClean):
	# Remove beginning and ending whitespace
	string = stringToClean.strip()
	# Replace html
	cleanString = h.unescape(string)
	# Replace unicode
	cleanString2 = unicodedata.normalize('NFKD', cleanString)
	return cleanString2


# Finds the cafeteria id and name
def getBonAppMenuData(url, id):
	params = {'cafe': id}
	response = requests.get(url, params=params)
	data = response.text

	try:
		# Grab cafe Id from JSON
		cafeId = list(json.loads(data)["cafes"].keys())[0]

		# Grab cafe name from JSON
		cafeName = json.loads(data)["cafes"][cafeId]["name"]


		# We want to display titles at least. We can do without location.
		try:
			# Grab cafe city from JSON
			cafeCity = json.loads(data)["cafes"][cafeId]["city"]
		except:
			cafeCity = ""
		try:
			# Grab cafe state from JSON
			cafeState = json.loads(data)["cafes"][cafeId]["state"]
		except:
			cafeState = ""


		# Formatting city and state strings
		# Both empty
		if cafeCity == "" and cafeState == "":
			cafeLoc = "No location listed"
		# Only city
		elif cafeCity != "" and cafeState == "":
			cafeLoc = clean(cafeCity)
		# Only State
		elif cafeCity == "" and cafeState != "":
			cafeLoc = clean(cafeState)
		# City and State
		else:
			cafeLoc = clean(cafeCity) + ", " + clean(cafeState)


		# Clean up the cafe name
		cafeName = clean(cafeName)

		# Construct the full return string
		appendData(cafeId, cafeName, cafeLoc)
	except:
		print('[Skipping. Moving on...]')
		pass

# Round numbers to a decimal point
def num2str(num, precision):
	return "%0.*f" % (precision, num)

# Get the outfile's size
def calculateFileSize():
	return str(os.path.getsize(fileName))


# Loop through the "known" amount of cafes
for num in range(0, totalCafes):
	getBonAppMenuData(url, num)
	time.sleep(3)

# Write our output to a file
with open(fileName, 'w') as outfile:
	# Output the data into a file
	json.dump(responseData, outfile)
	# Save the runtime
	endTime = time.time() - start;
{'fraction_non_alphanumeric': 0.12366096598383762, 'fraction_numerical': 0.004886299567750423, 'mean_word_length': 4.102588686481304, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7086084', 'n_tokens_mistral': 1999, 'n_tokens_neox': 1843, 'n_words': 555}