<commit_before>import sys
version = sys.version_info[0]
if version == 2:
	from urllib2 import Request, urlopen  # Python 2
	from urllib import urlretrieve
else:
	from urllib.request import Request, urlopen, urlretrieve  # Python 3


## python 3 version
# import urllib.request as request
# url = 'http://www.oschina.net'
# headers = ('User-Agent','Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11')
# opener = request.build_opener()
# opener.addheaders = [ headers ]
# origin_bytes = opener.open( url ).read()
# origin_string = origin_bytes.decode( 'utf-8' )
# print(origin_string)


url = 'http://yuedu.fm/'
header = {'User-agent' : 'Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5'}
req = Request(url, headers=header)
res = urlopen(req).read()
print(res)<commit_msg>add decode for python3<commit_after>import sys
version = sys.version_info[0]
if version == 2:
	from urllib2 import Request, urlopen  # Python 2
	from urllib import urlretrieve
else:
	from urllib.request import Request, urlopen, urlretrieve  # Python 3


## python 3 version
# import urllib.request as request
# url = 'http://www.oschina.net'
# headers = ('User-Agent','Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11')
# opener = request.build_opener()
# opener.addheaders = [ headers ]
# origin_bytes = opener.open( url ).read()
# origin_string = origin_bytes.decode( 'utf-8' )
# print(origin_string)


url = 'http://yuedu.fm/'
headers = {'User-agent' : 'Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5'}
req = Request(url, headers=headers)
res = urlopen(req).read().decode('utf-8')
print(type(res))
print(res)
{'fraction_non_alphanumeric': 0.15550510783200908, 'fraction_numerical': 0.056753688989784334, 'mean_word_length': 4.561514195583596, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7126624', 'n_tokens_mistral': 710, 'n_tokens_neox': 648, 'n_words': 161}