<commit_before>from __future__ import print_function
import os, json, pickle, re
import httplib, urllib
from _module import _module

class google( _module ):
	"""Bot module to search on google"""
	google_cache_file = os.path.expanduser( '~/.google_cache' )
	
	def __init__( self, config, bot ):
		_module.__init__( self, config, bot )

		if not os.path.exists( self.google_cache_file ):
			with open( self.google_cache_file, 'w' ):
				pass
		with open( self.google_cache_file, 'rb' ) as f:
			try:
				self.google_cache = pickle.load( f )
			except EOFError:
				self.google_cache = {}
				
	def can_handle( self, cmd, admin ):
		return self.api_key and self.cx and cmd == 'google' or ( admin and cmd == 'google_clear_cache' )
		
	def handle( self, bot, cmd, args, source, target, admin ):
		if not self.api_key and self.cx:
			return
		if cmd == 'help':
			self.bot.privmsg( target, '!google <query>: search on google' )
			return
		if admin and cmd == 'google_clear_cache':
			self.google_cache = {}
		if cmd == 'google':
			query = ' '.join( args )
		
			if not query in self.google_cache:
				conn = httplib.HTTPSConnection( 'www.googleapis.com' )
				conn.request(
					'GET',
					'/customsearch/v1?' + urllib.urlencode({
						'cx': self.cx,
						'key': self.api_key,
						'q': query
					})
				)
				response = conn.getresponse()
			
				self.google_cache[ query ] = json.load( response )
				with open( self.google_cache_file, 'wb' ) as f:
					pickle.dump( self.google_cache, f )
			
			results = self.google_cache[ query ]
			if 'items' in results and results['items'] and results['items'][0]:
				return [
					"{0}: {1}".format(
						results['items'][0]['title'],
						results['items'][0]['link']
					),
					re.sub( ' {2,}', ' ', results['items'][0]['snippet'] ),
					'Meer resultaten: http://www.google.nl/search?q={0}'.format( '+'.join( query ) )
				]<commit_msg>Actually fixed bug with google urls now<commit_after>from __future__ import print_function
import os, json, pickle, re
import httplib, urllib
from _module import _module

class google( _module ):
	"""Bot module to search on google"""
	google_cache_file = os.path.expanduser( '~/.google_cache' )
	
	def __init__( self, config, bot ):
		_module.__init__( self, config, bot )

		if not os.path.exists( self.google_cache_file ):
			with open( self.google_cache_file, 'w' ):
				pass
		with open( self.google_cache_file, 'rb' ) as f:
			try:
				self.google_cache = pickle.load( f )
			except EOFError:
				self.google_cache = {}
				
	def can_handle( self, cmd, admin ):
		return self.api_key and self.cx and cmd == 'google' or ( admin and cmd == 'google_clear_cache' )
		
	def handle( self, bot, cmd, args, source, target, admin ):
		if not self.api_key and self.cx:
			return
		if cmd == 'help':
			self.bot.privmsg( target, '!google <query>: search on google' )
			return
		if admin and cmd == 'google_clear_cache':
			self.google_cache = {}
		if cmd == 'google':
			query = ' '.join( args )
		
			if not query in self.google_cache:
				conn = httplib.HTTPSConnection( 'www.googleapis.com' )
				conn.request(
					'GET',
					'/customsearch/v1?' + urllib.urlencode({
						'cx': self.cx,
						'key': self.api_key,
						'q': query
					})
				)
				response = conn.getresponse()
			
				self.google_cache[ query ] = json.load( response )
				with open( self.google_cache_file, 'wb' ) as f:
					pickle.dump( self.google_cache, f )
			
			results = self.google_cache[ query ]
			if 'items' in results and results['items'] and results['items'][0]:
				return [
					"{0}: {1}".format(
						results['items'][0]['title'],
						results['items'][0]['link']
					),
					re.sub( ' {2,}', ' ', results['items'][0]['snippet'] ),
					'Meer resultaten: http://www.google.nl/search?q={0}'.format( '+'.join( args ) )
				]
{'fraction_non_alphanumeric': 0.15311760063141278, 'fraction_numerical': 0.004735595895816891, 'mean_word_length': 3.1281216069489686, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '64148', 'n_tokens_mistral': 1574, 'n_tokens_neox': 1300, 'n_words': 348}