<commit_before>import json

class Fetcher:

	def __init__(self, async = False):
		
		self.ASYNC = async


	def get(url, data):
		
		# Override in subclass
		raise NotImplementedException()


	def __get__(url, data, method, catch):

		try:
			json_results = method(url)

		except catch:
			raise ResolutionException('Could not connect to URL ' + url)

		else:
			
			try:
				return json.loads(json_results)

			except ValueError:
				raise ResolutionException('Could not parse JSON results')


class GoogleAppEngineFetcher(Fetcher):

	from google.appengine.api import urlfetch


	def __init__(self, async = False):

		super(GoogleAppEngineFetcher, self).__init__(async)


	def get(url, data):

		self.__get__(url, data, lambda u: urlfetch.fetch(url).contents, urlfetch.DownloadError)
		

class DefaultFetcher(Fetcher):

	import urllib

	def __init__(self, async = False):

		super(DefaultFetcher, self).__init__(async)

	def get(url, data):
		
		self.__get__(url, data, lambda u: urllib.urlopen(u).read(), IOError)

			




		<commit_msg>fixed environment specific imports in fetchers to be conditional<commit_after>import json

class Fetcher:

	def get(self, url, data):
		
		# Override in subclass
		raise NotImplementedException()


	def __get__(self, url, data, method, catch):

		try:
			json_results = method(url)

		except catch:
			raise ResolutionException('Could not connect to URL ' + url)

		else:
			
			try:
				return json.loads(json_results)

			except ValueError:
				raise ResolutionException('Could not parse JSON results')


class GoogleAppEngineFetcher(Fetcher):

	try:
		from google.appengine.api import urlfetch
	except ImportError:
		pass

	def get(self, url, data):

		return self.__get__(url, data, lambda u: urlfetch.fetch(url).contents, urlfetch.DownloadError)
		

class DefaultFetcher(Fetcher):

	try:
		import urllib
	except ImportError:
		pass

	def get(self, url, data):
		
		return self.__get__(url, data, lambda u: urllib.urlopen(u).read(), IOError)

			




		
{'fraction_non_alphanumeric': 0.09674185463659148, 'fraction_numerical': 0.0, 'mean_word_length': 3.9405940594059405, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2421933', 'n_tokens_mistral': 742, 'n_tokens_neox': 623, 'n_words': 178}