<commit_before>#!/usr/bin/python
# -*- coding:utf8 -*-

from bs4 import BeautifulSoup

import urllib2,re,os,argparse,sys

ALL_DOWNLOADS = 0
START_PAGE = 0
END_PAGE = 65589
BASE_DIR = u"images"
IMAGE_DIR_PATH = ''
KEEP_WORKING = True

def initArgs():
	global START_PAGE,END_PAGE,BASE_DIR
	parse = argparse.ArgumentParser()
	parse.add_argument('-s', '--startpage', dest='startpage', type=int, nargs='?', const=0, default=0, help='The first page to scrapy, default is 0.')
	parse.add_argument('-e', '--endpage', dest='endpage', type=int, nargs='?', const=65589, default=65589, help='The last page to scrapy, default is 65589.')
	parse.add_argument('-d', '--dir', dest='basedir', type=str, nargs='?', const=u"images", default=u"images", help='The base dir to save image, default is images')
	args = parse.parse_args()
	START_PAGE = args.startpage
	END_PAGE = args.endpage
	BASE_DIR = args.basedir
	print u"Save image to %s and Search from %s page to %s page..." % (BASE_DIR, START_PAGE, END_PAGE)	

def initBasePath():
	global BASE_DIR,IMAGE_DIR_PATH
	currentPath = os.getcwd()
	IMAGE_DIR_PATH = os.path.join(currentPath, BASE_DIR)
	if not os.path.isdir(IMAGE_DIR_PATH):
		os.mkdir(IMAGE_DIR_PATH)

def scrapyImages(page=1):
	global ALL_DOWNLOADS,START_PAGE,END_PAGE,BASE_DIR,IMAGE_DIR_PATH,KEEP_WORKING
	KEEP_WORKING = True
	while KEEP_WORKING:
		pageUrl = r"http://me2-sex.lofter.com/?page=%s" % page
		print pageUrl
		pageContent = urllib2.urlopen(pageUrl)
		pageSoup = BeautifulSoup(pageContent)
		imageGroup = pageSoup.find_all("a", class_="img", href=re.compile("post"))
		imageGroupCount = 0
		for groupItem in imageGroup:
			groupUrl = groupItem.get("href")
			print groupUrl
			imageGroupCount = imageGroupCount + 1
			groupContent = urllib2.urlopen(groupUrl)
			groupSoup = BeautifulSoup(groupContent)
			groupID = u"default"
			descriptMeta = groupSoup.find("meta", attrs={"name": "Description"})
			if descriptMeta is  None:
				groupID = u"default"
			else:
				groupID = descriptMeta["content"].strip()
				if len(groupID) == 0:
					groupID = u"default"
				else:
					groupID = groupID.split(" ")[0]
			print groupID
			targetDirPath = os.path.join(IMAGE_DIR_PATH, groupID)
			if not os.path.isdir(targetDirPath):
				os.mkdir(targetDirPath)
			images = groupSoup.find_all("div", class_="pic")
			for imageItem in images:
				imageUrl = imageItem.a.img.get("src")
				imageContent = urllib2.urlopen(imageUrl).read()
				imageSavePath = targetDirPath + "/" + imageUrl.split("/")[-1]
				if os.path.exists(imageSavePath):
					print "Image is existed:" + imageSavePath
					continue
				with open(imageSavePath, 'wb') as datas:
					datas.write(imageContent)
				ALL_DOWNLOADS = ALL_DOWNLOADS + 1
				print imageUrl
		print u"This page has %s image groups. " % imageGroupCount
		if imageGroupCount > 0:
			page = int(page) + 1
			if int(page) > END_PAGE:
				KEEP_WORKING = False
			else:
				print u"\nStart next page..."
		else:
			KEEP_WORKING = False
	else:
		print u"Prograss Done!"
		print u"All download %s images" % ALL_DOWNLOADS

initArgs()
initBasePath()
try:
	scrapyImages(START_PAGE)
except:
	print u"Some except out, prograss is stop."

<commit_msg>add thread and exception<commit_after>#!/usr/bin/python
# -*- coding:utf8 -*-

from bs4 import BeautifulSoup

import urllib2,re,os,argparse,thread,time

ALL_DOWNLOADS = 0
START_PAGE = 0
END_PAGE = 65589
BASE_DIR = u"images"
IMAGE_DIR_PATH = ''
KEEP_WORKING = True
MAX_PAGE_ERROR_TIMES = 3
CURRENT_PAGE_ERROR_TIMES = 0

def initArgs():
	global START_PAGE,END_PAGE,BASE_DIR
	parse = argparse.ArgumentParser()
	parse.add_argument('-s', '--startpage', dest='startpage', type=int, nargs='?', const=0, default=0, help='The first page to scrapy, default is 0.')
	parse.add_argument('-e', '--endpage', dest='endpage', type=int, nargs='?', const=65589, default=65589, help='The last page to scrapy, default is 65589.')
	parse.add_argument('-d', '--dir', dest='basedir', type=str, nargs='?', const=u"images", default=u"images", help='The base dir to save image, default is images')
	args = parse.parse_args()
	START_PAGE = args.startpage
	END_PAGE = args.endpage
	BASE_DIR = args.basedir
	print u"\n======================================================================="
	print u"Save image to %s and Search from %s page to %s page..." % (BASE_DIR, START_PAGE, END_PAGE)	

def initBasePath():
	global BASE_DIR,IMAGE_DIR_PATH
	currentPath = os.getcwd()
	IMAGE_DIR_PATH = os.path.join(currentPath, BASE_DIR)
	if not os.path.isdir(IMAGE_DIR_PATH):
		os.mkdir(IMAGE_DIR_PATH)

def inputHandle():
	global KEEP_WORKING
	print u"Now start scrapy prograss,you can type ENTER key to stop anytime..."
	print u"=======================================================================\n"
	raw_input()
	KEEP_WORKING = False
	print "Handle the working prograss, please wait...\n"
	
def scrapyImages(page=1):
	global ALL_DOWNLOADS,START_PAGE,END_PAGE,BASE_DIR,IMAGE_DIR_PATH,KEEP_WORKING,MAX_PAGE_ERROR_TIMES,CURRENT_PAGE_ERROR_TIMES
	KEEP_WORKING = True
	thread.start_new(inputHandle,())
	time.sleep(1)
	while KEEP_WORKING:
		pageUrl = r"http://me2-sex.lofter.com/?page=%s" % page
		print pageUrl
		try:
			pageContent = urllib2.urlopen(pageUrl)
			CURRENT_PAGE_ERROR_TIMES = 0
		except:
			CURRENT_PAGE_ERROR_TIMES = CURRENT_PAGE_ERROR_TIMES + 1
			if CURRENT_PAGE_ERROR_TIMES >= MAX_PAGE_ERROR_TIMES:
				print u"Get %s pages failed,please check your network or try later" % MAX_PAGE_ERROR_TIMES
				break
			else:
				print u"Get %s page failed, try next page...\n" % page
				continue
		pageSoup = BeautifulSoup(pageContent)
		imageGroup = pageSoup.find_all("a", class_="img", href=re.compile("post"))
		imageGroupCount = 0
		for groupItem in imageGroup:
			if not KEEP_WORKING:
				break
			groupUrl = groupItem.get("href")
			print groupUrl
			imageGroupCount = imageGroupCount + 1
			try:
				groupContent = urllib2.urlopen(groupUrl)
			except:
				print u"Get this group images fail, try next gourp..."
				continue
			groupSoup = BeautifulSoup(groupContent)
			groupID = u"default"
			descriptMeta = groupSoup.find("meta", attrs={"name": "Description"})
			if descriptMeta is  None:
				groupID = u"default"
			else:
				groupID = descriptMeta["content"].strip()
				if len(groupID) == 0:
					groupID = u"default"
				else:
					groupID = groupID.split(" ")[0]
			print groupID
			targetDirPath = os.path.join(IMAGE_DIR_PATH, groupID)
			if not os.path.isdir(targetDirPath):
				os.mkdir(targetDirPath)
			images = groupSoup.find_all("div", class_="pic")
			for imageItem in images:
				if not KEEP_WORKING:
					break
				imageUrl = imageItem.a.img.get("src")
				try:
					imageContent = urllib2.urlopen(imageUrl).read()
				except:
					print "Get image fail:" + imageUrl
					continue
				imageSavePath = targetDirPath + "/" + imageUrl.split("/")[-1]
				if os.path.exists(imageSavePath):
					print "Image is existed:" + imageSavePath
					continue
				try:
					with open(imageSavePath, 'wb') as datas:
						datas.write(imageContent)
				except:
					print "Save image fail:" + imageSavePath
					continue
				ALL_DOWNLOADS = ALL_DOWNLOADS + 1
				print "Save image success:" + imageSavePath
		print u"This page has %s image groups. " % imageGroupCount
		if imageGroupCount > 0 and KEEP_WORKING:
			page = int(page) + 1
			if int(page) > END_PAGE:
				KEEP_WORKING = False
			else:
				print u"-------------------------------------------"
				print u"Start next page..."
		else:
			KEEP_WORKING = False
	else:
		print u"======================================"
		print u"Prograss Done!"
		print u"All download %s images" % ALL_DOWNLOADS
		print u"======================================"

initArgs()
initBasePath()
try:
	scrapyImages(START_PAGE)
except KeyboardInterrupt:
	print "Unnatural exit.\nIf you want to cancel the progress,please type ENTER key."
except:
	print "Unkown exception."
	

{'fraction_non_alphanumeric': 0.1519857433808554, 'fraction_numerical': 0.011074338085539715, 'mean_word_length': 4.259036144578313, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2777799', 'n_tokens_mistral': 3106, 'n_tokens_neox': 2691, 'n_words': 652}