<commit_before>import asyncio
import discord
from   urllib.parse import quote
from   discord.ext import commands
from   Cogs import Settings
from   Cogs import DisplayName
from   Cogs import TinyURL
import urllib
import requests
import json
import os

class Search:

	# Init with the bot reference
	def __init__(self, bot, auth_file: str = None):
		self.bot = bot
		self.site_auth = None
		if os.path.isfile(auth_file):
		        with open(auth_file, 'r') as f:
		                self.site_auth = f.read()

	@commands.command(pass_context=True)
	async def google(self, ctx, *, query = None):
		"""Get some searching done."""

		if query == None:
			msg = 'You need a topic for me to Google.'
			await self.bot.send_message(ctx.message.channel, msg)
			return

		lmgtfy = "http://lmgtfy.com/?q={}".format(quote(query))
		lmgtfyT = TinyURL.tiny_url(lmgtfy)
		msg = '*{}*, you can find your answers here:\n\n{}'.format(DisplayName.name(ctx.message.author), lmgtfyT)
		# Say message
		await self.bot.send_message(ctx.message.channel, msg)

	@commands.command(pass_context=True)
	async def bing(self, ctx, *, query = None):
		"""Get some uh... more searching done."""

		if query == None:
			msg = 'You need a topic for me to Bing.'
			await self.bot.send_message(ctx.message.channel, msg)
			return

		lmgtfy = "http://letmebingthatforyou.com/?q={}".format(quote(query))
		lmgtfyT = TinyURL.tiny_url(lmgtfy)
		msg = '*{}*, you can find your answers here:\n\n{}'.format(DisplayName.name(ctx.message.author), lmgtfyT)
		# Say message
		await self.bot.send_message(ctx.message.channel, msg)

	@commands.command(pass_context=True)
	async def duck(self, ctx, *, query = None):
		"""Duck Duck... GOOSE."""

		if query == None:
			msg = 'You need a topic for me to DuckDuckGo.'
			await self.bot.send_message(ctx.message.channel, msg)
			return

		lmgtfy = "https://lmddgtfy.net/?q={}".format(quote(query))
		lmgtfyT = TinyURL.tiny_url(lmgtfy)
		msg = '*{}*, you can find your answers here:\n\n{}'.format(DisplayName.name(ctx.message.author), lmgtfyT)
		# Say message
		await self.bot.send_message(ctx.message.channel, msg)


	@commands.command(pass_context=True)
	async def searchsite(self, ctx, category_name = None, *, query = None):
		"""Search corpnewt.com forums."""

		auth = self.site_auth

		if auth == None:
                        await self.bot.say("Sorry this feature is not supported!")
                        return

		if query == None or category_name == None:
			msg = "Usage: $searchsite [category] [search term]\n\n Categories can be found at:\n\nhttps://corpnewt.com/"
			await self.bot.send_message(ctx.message.channel, msg)
			return

		categories_url = "https://corpnewt.com/api/categories"
		r = requests.get(categories_url, headers={'Authorization': auth})
		categories_json = json.loads(r.text)
		categories = categories_json["categories"]

		category = await self.find_category(categories, category_name)

		if category == None:
			await self.bot.say("Usage: $searchsite [category] [search term]\n\n Categories can be found at:\n\nhttps://corpnewt.com/")
			return

		search_url = "https://corpnewt.com/api/search?term={}&in=titlesposts&categories[]={}&searchChildren=true&showAs=posts".format(query, category["cid"])
		r = requests.get(search_url, headers={'Authorization': auth})
		search_json = json.loads(r.text)
		posts = search_json["posts"]
		result_string = '**Found {} Results for: {}**\n\n'.format(len(posts), query)
		limit = 5
		ctr = 0
		for post in posts:
                        if ctr < limit:
                                ctr = ctr + 1
                                result_string += '```{}``` \n\n https://corpnewt.com/{}\n\n'.format(post["topic"]["title"], post["topic"]["slug"])<commit_msg>Small typo in searchsite link output<commit_after>import asyncio
import discord
from   urllib.parse import quote
from   discord.ext import commands
from   Cogs import Settings
from   Cogs import DisplayName
from   Cogs import TinyURL
import urllib
import requests
import json
import os

class Search:

	# Init with the bot reference
	def __init__(self, bot, auth_file: str = None):
		self.bot = bot
		self.site_auth = None
		if os.path.isfile(auth_file):
		        with open(auth_file, 'r') as f:
		                self.site_auth = f.read()

	@commands.command(pass_context=True)
	async def google(self, ctx, *, query = None):
		"""Get some searching done."""

		if query == None:
			msg = 'You need a topic for me to Google.'
			await self.bot.send_message(ctx.message.channel, msg)
			return

		lmgtfy = "http://lmgtfy.com/?q={}".format(quote(query))
		lmgtfyT = TinyURL.tiny_url(lmgtfy)
		msg = '*{}*, you can find your answers here:\n\n{}'.format(DisplayName.name(ctx.message.author), lmgtfyT)
		# Say message
		await self.bot.send_message(ctx.message.channel, msg)

	@commands.command(pass_context=True)
	async def bing(self, ctx, *, query = None):
		"""Get some uh... more searching done."""

		if query == None:
			msg = 'You need a topic for me to Bing.'
			await self.bot.send_message(ctx.message.channel, msg)
			return

		lmgtfy = "http://letmebingthatforyou.com/?q={}".format(quote(query))
		lmgtfyT = TinyURL.tiny_url(lmgtfy)
		msg = '*{}*, you can find your answers here:\n\n{}'.format(DisplayName.name(ctx.message.author), lmgtfyT)
		# Say message
		await self.bot.send_message(ctx.message.channel, msg)

	@commands.command(pass_context=True)
	async def duck(self, ctx, *, query = None):
		"""Duck Duck... GOOSE."""

		if query == None:
			msg = 'You need a topic for me to DuckDuckGo.'
			await self.bot.send_message(ctx.message.channel, msg)
			return

		lmgtfy = "https://lmddgtfy.net/?q={}".format(quote(query))
		lmgtfyT = TinyURL.tiny_url(lmgtfy)
		msg = '*{}*, you can find your answers here:\n\n{}'.format(DisplayName.name(ctx.message.author), lmgtfyT)
		# Say message
		await self.bot.send_message(ctx.message.channel, msg)


	@commands.command(pass_context=True)
	async def searchsite(self, ctx, category_name = None, *, query = None):
		"""Search corpnewt.com forums."""

		auth = self.site_auth

		if auth == None:
                        await self.bot.say("Sorry this feature is not supported!")
                        return

		if query == None or category_name == None:
			msg = "Usage: $searchsite [category] [search term]\n\n Categories can be found at:\n\nhttps://corpnewt.com/"
			await self.bot.send_message(ctx.message.channel, msg)
			return

		categories_url = "https://corpnewt.com/api/categories"
		r = requests.get(categories_url, headers={'Authorization': auth})
		categories_json = json.loads(r.text)
		categories = categories_json["categories"]

		category = await self.find_category(categories, category_name)

		if category == None:
			await self.bot.say("Usage: $searchsite [category] [search term]\n\n Categories can be found at:\n\nhttps://corpnewt.com/")
			return

		search_url = "https://corpnewt.com/api/search?term={}&in=titlesposts&categories[]={}&searchChildren=true&showAs=posts".format(query, category["cid"])
		r = requests.get(search_url, headers={'Authorization': auth})
		search_json = json.loads(r.text)
		posts = search_json["posts"]
		result_string = '**Found {} Results for: {}**\n\n'.format(len(posts), query)
		limit = 5
		ctr = 0
		for post in posts:
                        if ctr < limit:
                                ctr = ctr + 1
                                result_string += '```{}``` \n\n https://corpnewt.com/topic/{}\n\n'.format(post["topic"]["title"], post["topic"]["slug"])
{'fraction_non_alphanumeric': 0.14564143639033506, 'fraction_numerical': 0.0008009611533840609, 'mean_word_length': 3.4943011397720456, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 12, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1945598', 'n_tokens_mistral': 2678, 'n_tokens_neox': 2467, 'n_words': 606}