<commit_before>#!/usr/bin/env python3
import string, os, time, json, re, unicodedata, html.parser, requests

# Bon appetit cafe hours api url
url = "http://legacy.cafebonappetit.com/api/2/cafes"
# How many cafeterias you want to parse (in order)
totalCafes = 10
# What our file should be named
fileName = "data.json"


# Our constructed JSON data
responseData = []
# Parser to clean up the string names
h = html.parser.HTMLParser()
# Time when we started the script
start = time.time()

def appendData(cafeId, cafeName, cafeLoc):
	print(cafeId + ") " + cafeName + " in " + cafeLoc)
	responseData.append({'id':cafeId, 'label':cafeName, 'desc':cafeLoc})


def clean(stringToClean):
	# Remove beginning and ending whitespace
	string = stringToClean.strip()
	# Replace html
	cleanString = h.unescape(string)
	# Replace unicode
	return unicodedata.normalize('NFKD', cleanString)


# Finds the cafeteria id and name
def getBonAppMenuData(url, id):
	response = requests.get(url, params={'cafe': id})
	data = response.json()<commit_msg>Up the amount of cafe metadata requests<commit_after>#!/usr/bin/env python3
import string, os, time, json, re, unicodedata, html.parser, requests

# Bon appetit cafe hours api url
url = "http://legacy.cafebonappetit.com/api/2/cafes"
# How many cafeterias you want to parse (in order)
totalCafes = 1818
# What our file should be named
fileName = "data.json"


# Our constructed JSON data
responseData = []
# Parser to clean up the string names
h = html.parser.HTMLParser()
# Time when we started the script
start = time.time()

def appendData(cafeId, cafeName, cafeLoc):
	print(cafeId + ") " + cafeName + " in " + cafeLoc)
	responseData.append({'id':cafeId, 'label':cafeName, 'desc':cafeLoc})


def clean(stringToClean):
	# Remove beginning and ending whitespace
	string = stringToClean.strip()
	# Replace html
	cleanString = h.unescape(string)
	# Replace unicode
	return unicodedata.normalize('NFKD', cleanString)


# Finds the cafeteria id and name
def getBonAppMenuData(url, id):
	response = requests.get(url, params={'cafe': id})
	data = response.json()
{'fraction_non_alphanumeric': 0.11900191938579655, 'fraction_numerical': 0.0047984644913627635, 'mean_word_length': 4.974212034383954, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4581747', 'n_tokens_mistral': 724, 'n_tokens_neox': 695, 'n_words': 213}