<commit_before>import json
import aiohttp


async def api_request(url, *, headers=None):
	"""
	Returns a JSON dict object for most api calls in RoxBot.
	:param url: URL Should be a api endpoint that will return
	:param headers: There is no need to pass the user agent, this is done for you.
	:return: dict of JSON or None if a JSON was not returned from the call.
	"""
	if headers is None:
		headers = {'User-agent': 'RoxBot Discord Bot'}
	else:
		headers = {'User-agent': 'RoxBot Discord Bot', **headers}
	async with aiohttp.ClientSession() as session:
		async with session.get(url, headers=headers) as resp:
			try:
				return json.loads(await resp.read())
			except json.JSONDecodeError:
				return None


async def download_file(url, filename=None):
	"""
	Downloads the file at the given url and then saves it under the filename given to disk.
	:param filename:
	:param url:
	"""
	if filename is None:
		filename = url.split("/")[-1]
	async with aiohttp.ClientSession() as session:
		async with session.get(url, headers={'User-agent': 'RoxBot Discord Bot'}) as data:
			with open(filename, 'wb') as f:
				f.write(await data.read())
	return filename


async def upload_file(url, file):
	"""

	:param url: url to POST to.
	:param file: Byes-like object to upload.
	:return:
	"""
	async with aiohttp.ClientSession() as session:
			payload = {"files": open(file, "rb")}
			resp = await session.post(url, headers={'User-agent': 'RoxBot Discord Bot', "content_type": "multipart/form-data"}, data=payload)
			return await resp.json()


async def get_page(url):
	"""
	Returns the page at the given url
	:param url: the url of the page you want to get
	:return: the html page
	"""
	async with aiohttp.ClientSession() as session:
		async with session.get(url, headers={'User-agent': 'RoxBot Discord Bot'}) as page:
			return await page.text()
<commit_msg>added a return for filename in the filedownloaded for roxbot.<commit_after>import json
import aiohttp


async def api_request(url, *, headers=None):
	"""
	Returns a JSON dict object for most api calls in RoxBot.
	:param url: URL Should be a api endpoint that will return
	:param headers: There is no need to pass the user agent, this is done for you.
	:return: dict of JSON or None if a JSON was not returned from the call.
	"""
	if headers is None:
		headers = {'User-agent': 'RoxBot Discord Bot'}
	else:
		headers = {'User-agent': 'RoxBot Discord Bot', **headers}
	async with aiohttp.ClientSession() as session:
		async with session.get(url, headers=headers) as resp:
			try:
				return json.loads(await resp.read())
			except json.JSONDecodeError:
				return None


async def download_file(url, filename=None):
	"""
	Downloads the file at the given url and then saves it under the filename given to disk.
	:param filename:
	:param url:
	"""
	if filename is None:
		filename = url.split("/")[-1].split("?")[0]
	async with aiohttp.ClientSession() as session:
		async with session.get(url, headers={'User-agent': 'RoxBot Discord Bot'}) as data:
			with open(filename, 'wb') as f:
				f.write(await data.read())
	return filename


async def upload_file(url, file):
	"""

	:param url: url to POST to.
	:param file: Byes-like object to upload.
	:return:
	"""
	async with aiohttp.ClientSession() as session:
			payload = {"files": open(file, "rb")}
			resp = await session.post(url, headers={'User-agent': 'RoxBot Discord Bot', "content_type": "multipart/form-data"}, data=payload)
			return await resp.json()


async def get_page(url):
	"""
	Returns the page at the given url
	:param url: the url of the page you want to get
	:return: the html page
	"""
	async with aiohttp.ClientSession() as session:
		async with session.get(url, headers={'User-agent': 'RoxBot Discord Bot'}) as page:
			return await page.text()

{'fraction_non_alphanumeric': 0.11935140882509304, 'fraction_numerical': 0.0007974481658692185, 'mean_word_length': 4.140710382513661, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2171076', 'n_tokens_mistral': 1260, 'n_tokens_neox': 1201, 'n_words': 471}