<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import urllib
import urllib2
import re
import ssl
import os
import platform
import sys

class Tools :

	def __init__ (self) :
		pass

	def getPage (self, url, requestHeader = [], postData = {}) :
		if postData == {} :
			request = urllib2.Request(url)
		elif isinstance(postData, basestring) :
			request = urllib2.Request(url, postData)
		else :
			request = urllib2.Request(url, urllib.urlencode(postData))

		for x in requestHeader :
			headerType = x.split(':')[0]
			headerCon = x.replace(headerType + ':', '')
			request.add_header(headerType, headerCon)

		try : 
			ctx = ssl.create_default_context()
			ctx.check_hostname = False
			ctx.verify_mode = ssl.CERT_NONE
			response = urllib2.urlopen(request, context = ctx)
			header = response.headers
			body = response.read()
			code = response.code
		except urllib2.HTTPError, e:
			header = e.headers
			body = e.read()
			code = e.code

		result = {
			'code': code,
			'header': header,
			'body': body
		}

		return result

	def fmtCookie (self, string) :
		result = re.sub(r"path\=\/.", "", string)
		result = re.sub(r"(\S*?)\=deleted.", "", result)
		result = re.sub(r"expires\=(.*?)GMT;", "", result)
		result = re.sub(r"domain\=(.*?)tv.", "", result)
		result = re.sub(r"httponly", "", result)
		result = re.sub(r"\s", "", result)

		return result

	def urlencode(self, str) :
		reprStr = repr(str).replace(r'\x', '%')
		return reprStr[1:-1]

	def getRes (self, fileName) :
		if getattr(sys, 'frozen', False):
			base_path = os.path.join(sys._MEIPASS, 'RES')
		else:
			base_path = os.path.join(os.path.abspath("../"), 'Resources')<commit_msg>Old style exception --> new style for Python 2 & 3<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import urllib
import urllib2
import re
import ssl
import os
import platform
import sys

class Tools :

	def __init__ (self) :
		pass

	def getPage (self, url, requestHeader = [], postData = {}) :
		if postData == {} :
			request = urllib2.Request(url)
		elif isinstance(postData, basestring) :
			request = urllib2.Request(url, postData)
		else :
			request = urllib2.Request(url, urllib.urlencode(postData))

		for x in requestHeader :
			headerType = x.split(':')[0]
			headerCon = x.replace(headerType + ':', '')
			request.add_header(headerType, headerCon)

		try : 
			ctx = ssl.create_default_context()
			ctx.check_hostname = False
			ctx.verify_mode = ssl.CERT_NONE
			response = urllib2.urlopen(request, context = ctx)
			header = response.headers
			body = response.read()
			code = response.code
		except urllib2.HTTPError as e:
			header = e.headers
			body = e.read()
			code = e.code

		result = {
			'code': code,
			'header': header,
			'body': body
		}

		return result

	def fmtCookie (self, string) :
		result = re.sub(r"path\=\/.", "", string)
		result = re.sub(r"(\S*?)\=deleted.", "", result)
		result = re.sub(r"expires\=(.*?)GMT;", "", result)
		result = re.sub(r"domain\=(.*?)tv.", "", result)
		result = re.sub(r"httponly", "", result)
		result = re.sub(r"\s", "", result)

		return result

	def urlencode(self, str) :
		reprStr = repr(str).replace(r'\x', '%')
		return reprStr[1:-1]

	def getRes (self, fileName) :
		if getattr(sys, 'frozen', False):
			base_path = os.path.join(sys._MEIPASS, 'RES')
		else:
			base_path = os.path.join(os.path.abspath("../"), 'Resources')
{'fraction_non_alphanumeric': 0.1707605800532702, 'fraction_numerical': 0.006510802012429713, 'mean_word_length': 3.4473684210526314, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6456426', 'n_tokens_mistral': 1387, 'n_tokens_neox': 1238, 'n_words': 278}