<commit_before>import urllib, urllib2, HTMLParser, re, platform, sys, os.path, socket
from datetime import datetime
#from socket import error

# Kink.com
DEBUG = False
KINK_ID_REGEX = "^[0-9]+"
KINK_BASE_SHOOT_URL = "http://www.kink.com/shoot/%s"
KINK_BASE_MODEL_URL = "http://www.kink.com/model/%s"
KINK_BASE_RATING_URL = "http://www.kink.com/api/ratings/%s"
KINK_HEADERS = {"User-Agent" : "Plex Agent"}

SITE_DIRS = []
ACTORS = []

missing_actors = {}

####################################################################################################
def PerformKinkSearch(shoot_id):
	data = {"id" : shoot_id}
	
	shoot_url = KINK_BASE_SHOOT_URL % shoot_id
	data["url"] = shoot_url
	
	request = urllib2.Request(shoot_url, headers=KINK_HEADERS)
	response = urllib2.urlopen(request)
	response_long = response.read()
	response = response_long.replace('\n', "")
	response = response.replace('\r', "")
	
	# Uncomment to spit out the HTML that's being worked with.
	#f = open(os.path.join(os.path.expanduser("~"), shoot_id + ".txt"), "w")
	#f.write(response_long)
	#f.close()
	
	data["title"] = PerformTitleSearch(response)
	data["date"] = PerformDateSearch(response)
	data["summary"] = PerformSummarySearch(response_long)
	data["actors"] = PerformActorSearch(response, shoot_id)
	data["tags"] = PerformTagSearch(response)
	data["rating"] = PerformRatingSearch(shoot_id)
	data["studio"] = PerformStudioSearch(response)
	data["cover"] = PerformCoverArtSearch(response)
	
	data["id"] = data["id"].zfill(5)
	
	return data
	
def PerformKinkUpdate(shoot_id):
	data = {"id" : shoot_id}
	
	shoot_url = KINK_BASE_SHOOT_URL % shoot_id
	data["url"] = shoot_url
	
	request = urllib2.Request(shoot_url, headers=KINK_HEADERS)
	response = urllib2.urlopen(request)
	response_long = response.read()
	response = response_long.replace('\n', "")
	response = response.replace('\r', "")
	
	data["rating"] = PerformRatingSearch(shoot_id)
	data["cover"] = PerformCoverArtSearch(response)
	
	data["id"] = data["id"].zfill(5)
	
	return data
	
def PerformTitleSearch(response):
	# Title is contained in a <h1> with the class "shoot-title"
	title_regex = "<h1 class=\"shoot-title\">(.*?)</h1>"
	match = re.search(title_regex, response)
	if match is not None:
		title = match.group(1).title().strip()
		title = title.replace("'S", "'s")
		title = title.replace("n'T", "n't")<commit_msg>Exceptions improved across Kink-related methods<commit_after>import urllib, urllib2, HTMLParser, re, platform, sys, os.path, socket
from datetime import datetime
#from socket import error

# Kink.com
DEBUG = False
KINK_ID_REGEX = "^[0-9]+"
KINK_BASE_SHOOT_URL = "http://www.kink.com/shoot/%s"
KINK_BASE_MODEL_URL = "http://www.kink.com/model/%s"
KINK_BASE_RATING_URL = "http://www.kink.com/api/ratings/%s"
KINK_HEADERS = {"User-Agent" : "Kink Util"}

SITE_DIRS = []
ACTORS = []

missing_actors = {}

####################################################################################################
def PerformKinkSearch(shoot_id):
	data = {"id" : shoot_id}
	
	shoot_url = KINK_BASE_SHOOT_URL % shoot_id
	data["url"] = shoot_url
	
	request = None
	response = None
	try:
		request = urllib2.Request(shoot_url, headers=KINK_HEADERS)
		response = urllib2.urlopen(request)
	except urllib2.HTTPError as e:
		raise IOError("HTTPError while opening shoot page.\nError: " + str(e))
	except urllib2.URLError as e:
		raise IOError("URLError while opening shoot page.\nError: " + str(e))
	response_long = response.read()
	response = response_long.replace('\n', "")
	response = response.replace('\r', "")
	
	# Uncomment to spit out the HTML that's being worked with.
	#f = open(os.path.join(os.path.expanduser("~"), shoot_id + ".txt"), "w")
	#f.write(response_long)
	#f.close()
	
	stage = None
	try:
		stage = "title"
		data["title"] = PerformTitleSearch(response)
		stage = "date"
		data["date"] = PerformDateSearch(response)
		stage = "summary"
		data["summary"] = PerformSummarySearch(response_long)
		stage = "actors"
		data["actors"] = PerformActorSearch(response, shoot_id)
		stage = "tags"
		data["tags"] = PerformTagSearch(response)
		stage = "rating"
		data["rating"] = PerformRatingSearch(shoot_id)
		stage = "studio"
		data["studio"] = PerformStudioSearch(response)
		stage = "cover"
		data["cover"] = PerformCoverArtSearch(response)
	except re.error as e:
		raise IOError("Regular expression error raised in " + stage + " stage.\nError: " + str(e) + 
		"\n\nHTML:" + response)
	except urllib2.HTTPError as e:
		raise IOError("HTTPError raised during stage" + stage + ".\nError: " + str(e))
	except urllib2.URLError as e:
		raise IOError("URLError raised during stage" + stage + ".\nError: " + str(e))
		
	data["id"] = data["id"].zfill(5)
	
	return data
	
def PerformKinkUpdate(shoot_id):
	data = {"id" : shoot_id}
	
	shoot_url = KINK_BASE_SHOOT_URL % shoot_id
	data["url"] = shoot_url
	
	request = urllib2.Request(shoot_url, headers=KINK_HEADERS)
	response = urllib2.urlopen(request)
	response_long = response.read()
	response = response_long.replace('\n', "")
	response = response.replace('\r', "")
	
	data["rating"] = PerformRatingSearch(shoot_id)
	data["cover"] = PerformCoverArtSearch(response)
	
	data["id"] = data["id"].zfill(5)
	
	return data
	
def PerformTitleSearch(response):
	# Title is contained in a <h1> with the class "shoot-title"
	title_regex = "<h1 class=\"shoot-title\">(.*?)</h1>"
	match = re.search(title_regex, response)
	if match is not None:
		title = match.group(1).title().strip()
		title = title.replace("'S", "'s")
		title = title.replace("n'T", "n't")
{'fraction_non_alphanumeric': 0.18941856265799928, 'fraction_numerical': 0.005417118093174431, 'mean_word_length': 5.287173666288309, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5304886', 'n_tokens_mistral': 2071, 'n_tokens_neox': 1951, 'n_words': 396}