<commit_before>#!/usr/bin/env python

#####################

import imghdr
import os
import re
import shutil
import tempfile
import threading
import time
import urllib2
import zipfile

#####################

try:
	import urllib
except ImportError:
	import urllib.parse as urllib

#####################

from util import * 

#####################

UA = """Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36"""

#####################

class SiteParserBase:

#####	
	# typical misspelling of title and/or manga removal
	class MangaNotFound(Exception):
		
		def __init__(self, errorMsg=''):
			self.errorMsg = 'Manga not found. %s' % errorMsg
		
		def __str__(self):
			return self.errorMsg
	
	# XML file config reports nothing to do
	class NoUpdates(Exception):

		def __init__(self, errorMsg=''):
			self.errorMsg = 'No updates. %s' % errorMsg

		def __str__(self):
			return self.errorMsg
		
#####

	def __init__(self,optDict):
		for elem in vars(optDict):
			setattr(self, elem, getattr(optDict, elem))
		self.chapters = []
		self.chapters_to_download = []
		self.tempFolder = tempfile.mkdtemp()
		self.garbageImages = {}
		
		# should be defined by subclasses
		self.re_getImage = None
		self.re_getMaxPages = None
		self.isPrependMangaName = False

	# this takes care of removing the temp directory after the last successful download
	def __del__(self):
		try:
			shutil.rmtree(self.tempFolder)
		except:
			pass
		if len(self.garbageImages) > 0:
			print('\nSome images were not downloaded due to unavailability on the site or temporary ip banning:\n')
			for elem in self.garbageImages.keys():
				print('Manga keyword: %s' % elem)
				print('Pages: %s' % self.garbageImages[elem])
#####	
		
	def downloadChapter(self):
		raise NotImplementedError('Should have implemented this')		
		
	def parseSite(self):
		raise NotImplementedError('Should have implemented this')
#####
	
	def compress(self, mangaChapterPrefix, max_pages):
		"""
		Looks inside the temporary directory and zips up all the image files.
		"""
		if self.verbose_FLAG:
			print('Compressing...')
		
		compressedFile = os.path.join(self.tempFolder, mangaChapterPrefix) + self.downloadFormat
			
		z = zipfile.ZipFile( compressedFile, 'w')
		
		for page in range(1, max_pages + 1):	
			tempPath = os.path.join(self.tempFolder, mangaChapterPrefix + '_' + str(page).zfill(3))
			# we got an image file
			if os.path.exists(tempPath) is True and imghdr.what(tempPath) != None:
				z.write( tempPath, mangaChapterPrefix + '_' + str(page).zfill(3) + '.' + imghdr.what(tempPath))
			# site has thrown a 404 because image unavailable or using anti-leeching
			else:
				if mangaChapterPrefix not in self.garbageImages:
					self.garbageImages[mangaChapterPrefix] = [page]
				else:
					self.garbageImages[mangaChapterPrefix].append(page)
				
		z.close()
		
		if self.overwrite_FLAG == True:
			priorPath = os.path.join(self.downloadPath, mangaChapterPrefix) + self.downloadFormat
			if os.path.exists(priorPath):
				os.remove(priorPath)
		
		shutil.move(compressedFile, self.downloadPath)
		
		# The object conversionQueue (singleton) stores the path to every compressed file that  
		# has been downloaded. This object is used by the conversion code to convert the downloaded images
		# to the format specified by the Device errorMsg
		
		compressedFile = os.path.basename(compressedFile)
		compressedFile = os.path.join(self.downloadPath, compressedFile)
		return compressedFile
	
	def downloadImage(self, downloadThread, page, pageUrl, manga_chapter_prefix):
		"""
		Given a page URL to download from, it searches using self.imageRegex
		to parse out the image URL, and downloads and names it using 
		manga_chapter_prefix and page.
		"""

		# while loop to protect against server denies for requests
		# note that disconnects are already handled by getSourceCode, we use a 
		# regex to parse out the image URL and filter out garbage denies
		maxRetries = 5
		waitRetryTime = 5
		while True:
			try:
				if (self.verbose_FLAG):
					print(pageUrl)
				source_code = getSourceCode(pageUrl, self.proxy)
				img_url = self.__class__.re_getImage.search(source_code).group(1)
				if (self.verbose_FLAG):
					print("Image URL: %s" % img_url)
			except AttributeError:
				if (maxRetries == 0):
					if (not self.verbose_FLAG):
						self.outputMgr.updateOutputObj( downloadThread.outputIdx )
					return	
				else:
					# random dist. for further protection against anti-leech
					# idea from wget
					time.sleep(random.uniform(0.5*waitRetryTime, 1.5*waitRetryTime))
					maxRetries -= 1
			else:
				break

		# Remove the 'http://' before encoding, otherwise the '://' would be 
		# encoded as well				
#		img_url = 'http://' + urllib.quote(img_url.split('//')[1])
		
		if self.verbose_FLAG:
			print(img_url)
		
		# Loop to protect against server denies for requests and/or minor disconnects
		maxRetries = 5
		while True:
			if (maxRetries == 0):
				break;

			response = None
			try:
				request = urllib2.Request(img_url, headers={'User-Agent': UA})
				response = urllib2.urlopen(request, timeout=45)
				responseInfo = response.info()
				contentLength = int(responseInfo["Content-Length"])

				temp_path = os.path.join(self.tempFolder, manga_chapter_prefix + '_' + str(page).zfill(3))
				f = open(temp_path, 'w')

				f.write(response.read())
				response.close()

				f.close()

				# Check file size
				fileSize = os.path.getsize(temp_path)
				if fileSize < contentLength:
					if self.verbose_FLAG:
						print("File Size (" + str(fileSize) + ") < Content Length (" + str(contentLength) + ")")
					maxRetries -= 1
					continue

			except IOError:
				if f:
					f.close()<commit_msg>Fixed crash when handling exception. (Regression caused by latest changes)<commit_after>#!/usr/bin/env python

#####################

import imghdr
import os
import re
import shutil
import tempfile
import threading
import time
import urllib2
import zipfile

#####################

try:
	import urllib
except ImportError:
	import urllib.parse as urllib

#####################

from util import * 

#####################

UA = """Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36"""

#####################

class SiteParserBase:

#####	
	# typical misspelling of title and/or manga removal
	class MangaNotFound(Exception):
		
		def __init__(self, errorMsg=''):
			self.errorMsg = 'Manga not found. %s' % errorMsg
		
		def __str__(self):
			return self.errorMsg
	
	# XML file config reports nothing to do
	class NoUpdates(Exception):

		def __init__(self, errorMsg=''):
			self.errorMsg = 'No updates. %s' % errorMsg

		def __str__(self):
			return self.errorMsg
		
#####

	def __init__(self,optDict):
		for elem in vars(optDict):
			setattr(self, elem, getattr(optDict, elem))
		self.chapters = []
		self.chapters_to_download = []
		self.tempFolder = tempfile.mkdtemp()
		self.garbageImages = {}
		
		# should be defined by subclasses
		self.re_getImage = None
		self.re_getMaxPages = None
		self.isPrependMangaName = False

	# this takes care of removing the temp directory after the last successful download
	def __del__(self):
		try:
			shutil.rmtree(self.tempFolder)
		except:
			pass
		if len(self.garbageImages) > 0:
			print('\nSome images were not downloaded due to unavailability on the site or temporary ip banning:\n')
			for elem in self.garbageImages.keys():
				print('Manga keyword: %s' % elem)
				print('Pages: %s' % self.garbageImages[elem])
#####	
		
	def downloadChapter(self):
		raise NotImplementedError('Should have implemented this')		
		
	def parseSite(self):
		raise NotImplementedError('Should have implemented this')
#####
	
	def compress(self, mangaChapterPrefix, max_pages):
		"""
		Looks inside the temporary directory and zips up all the image files.
		"""
		if self.verbose_FLAG:
			print('Compressing...')
		
		compressedFile = os.path.join(self.tempFolder, mangaChapterPrefix) + self.downloadFormat
			
		z = zipfile.ZipFile( compressedFile, 'w')
		
		for page in range(1, max_pages + 1):	
			tempPath = os.path.join(self.tempFolder, mangaChapterPrefix + '_' + str(page).zfill(3))
			# we got an image file
			if os.path.exists(tempPath) is True and imghdr.what(tempPath) != None:
				z.write( tempPath, mangaChapterPrefix + '_' + str(page).zfill(3) + '.' + imghdr.what(tempPath))
			# site has thrown a 404 because image unavailable or using anti-leeching
			else:
				if mangaChapterPrefix not in self.garbageImages:
					self.garbageImages[mangaChapterPrefix] = [page]
				else:
					self.garbageImages[mangaChapterPrefix].append(page)
				
		z.close()
		
		if self.overwrite_FLAG == True:
			priorPath = os.path.join(self.downloadPath, mangaChapterPrefix) + self.downloadFormat
			if os.path.exists(priorPath):
				os.remove(priorPath)
		
		shutil.move(compressedFile, self.downloadPath)
		
		# The object conversionQueue (singleton) stores the path to every compressed file that  
		# has been downloaded. This object is used by the conversion code to convert the downloaded images
		# to the format specified by the Device errorMsg
		
		compressedFile = os.path.basename(compressedFile)
		compressedFile = os.path.join(self.downloadPath, compressedFile)
		return compressedFile
	
	def downloadImage(self, downloadThread, page, pageUrl, manga_chapter_prefix):
		"""
		Given a page URL to download from, it searches using self.imageRegex
		to parse out the image URL, and downloads and names it using 
		manga_chapter_prefix and page.
		"""

		# while loop to protect against server denies for requests
		# note that disconnects are already handled by getSourceCode, we use a 
		# regex to parse out the image URL and filter out garbage denies
		maxRetries = 5
		waitRetryTime = 5
		while True:
			try:
				if (self.verbose_FLAG):
					print(pageUrl)
				source_code = getSourceCode(pageUrl, self.proxy)
				img_url = self.__class__.re_getImage.search(source_code).group(1)
				if (self.verbose_FLAG):
					print("Image URL: %s" % img_url)
			except AttributeError:
				if (maxRetries == 0):
					if (not self.verbose_FLAG):
						self.outputMgr.updateOutputObj( downloadThread.outputIdx )
					return	
				else:
					# random dist. for further protection against anti-leech
					# idea from wget
					time.sleep(random.uniform(0.5*waitRetryTime, 1.5*waitRetryTime))
					maxRetries -= 1
			else:
				break

		# Remove the 'http://' before encoding, otherwise the '://' would be 
		# encoded as well				
#		img_url = 'http://' + urllib.quote(img_url.split('//')[1])
		
		if self.verbose_FLAG:
			print(img_url)
		
		# Loop to protect against server denies for requests and/or minor disconnects
		maxRetries = 5
		while True:
			if (maxRetries == 0):
				break;

			response = None
			f = None
			try:
				request = urllib2.Request(img_url, headers={'User-Agent': UA})
				response = urllib2.urlopen(request, timeout=45)
				responseInfo = response.info()
				contentLength = int(responseInfo["Content-Length"])

				temp_path = os.path.join(self.tempFolder, manga_chapter_prefix + '_' + str(page).zfill(3))
				f = open(temp_path, 'w')

				f.write(response.read())
				response.close()

				f.close()

				# Check file size
				fileSize = os.path.getsize(temp_path)
				if fileSize < contentLength:
					if self.verbose_FLAG:
						print("File Size (" + str(fileSize) + ") < Content Length (" + str(contentLength) + ")")
					maxRetries -= 1
					continue

			except IOError:
				if f:
					f.close()
{'fraction_non_alphanumeric': 0.1216016676800139, 'fraction_numerical': 0.0093807000781725, 'mean_word_length': 3.6938442723196085, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1262374', 'n_tokens_mistral': 4365, 'n_tokens_neox': 3563, 'n_words': 1020}