<commit_before>import BaseHTTPServer
import SimpleHTTPServer
import sys
import os
import ssl

chunked_encoding = False

class http_server_with_timeout(BaseHTTPServer.HTTPServer):
	allow_reuse_address = True
	timeout = 120

	def handle_timeout(self):
		raise Exception('timeout')

class http_handler(SimpleHTTPServer.SimpleHTTPRequestHandler):

	def do_GET(s):

		#print s.requestline
		global chunked_encoding

		# if the request contains the hostname and port. strip it
		if s.path.startswith('http://') or s.path.startswith('https://'):
			s.path = s.path[8:]
			s.path = s.path[s.path.find('/'):]

		s.path = os.path.normpath(s.path)

		if s.path == '/redirect':
			s.send_response(301)
			s.send_header("Location", "/test_file")
			s.send_header("Connection", "close")
			s.end_headers()
		elif s.path == '/infinite_redirect':
			s.send_response(301)
			s.send_header("Location", "/infinite_redirect")
			s.send_header("Connection", "close")
			s.end_headers()
		elif s.path == '/relative/redirect':
			s.send_response(301)
			s.send_header("Location", "../test_file")
			s.send_header("Connection", "close")
			s.end_headers()
		elif s.path.startswith('/announce'):
			s.send_response(200)
			response = 'd8:intervali1800e8:completei1e10:incompletei1e5:peers0:e'
			s.send_header("Content-Length", "%d" % len(response))
			s.send_header("Connection", "close")
			s.end_headers()
			s.wfile.write(response)
		elif os.path.split(s.path)[1].startswith('seed?'):
			query = s.path[6:]
			args_raw = query.split('&')
			args = {}
			for a in args_raw:
				kvp = a.split('=')
				args[kvp[0]] = kvp[1]
			piece = int(args['piece'])
			ranges = args['ranges'].split('-')

			try:
				filename = s.path[1:s.path.find('seed?') + 4]
				#print 'filename = %s' % filename
				f = open(filename, 'rb')
				f.seek(piece * 64 * 1024 + int(ranges[0]))
				data = f.read(int(ranges[1]) - int(ranges[0]) + 1)
				f.close()

				s.send_response(200)
				print 'sending %d bytes' % len(data)
				s.send_header("Content-Length", "%d" % len(data))
				s.end_headers()
				s.wfile.write(data);
			except Exception, e:
				print 'FILE NOT FOUND: ', os.getcwd(), filename
				s.send_response(404)
				s.send_header("Content-Length", "0")
				s.end_headers()
		else:
			try:
				filename = s.path[1:]
				# serve file by invoking default handler
				f = open(filename)
				size = int(os.stat(filename).st_size)
				start_range = 0
				end_range = size
				if 'Range' in s.headers:
					s.send_response(206)
					s.send_header('Content-Range', 'bytes ' + str(start_range) + '-' + str(end_range - 1) + '/' + str(size))
					st, e = s.headers['range'][6:].split('-', 1)
					sl = len(st)
					el = len(e)
					if sl > 0:
						start_range = int(st)
						if el > 0:
							end_range = int(e) + 1
					elif el > 0:
						ei = int(e)
						if ei < size:
							start_range = size - ei
				else:
					s.send_response(200)
				s.send_header('Accept-Ranges', 'bytes')
				if chunked_encoding:
					s.send_header('Transfer-Encoding', 'chunked')
				s.send_header('Content-Length', end_range - start_range)
				if filename.endswith('.gz'):
					s.send_header('Content-Encoding', 'gzip')<commit_msg>open files in binary mode in unit test web server (may fix some issues on windows)<commit_after>import BaseHTTPServer
import SimpleHTTPServer
import sys
import os
import ssl

chunked_encoding = False

class http_server_with_timeout(BaseHTTPServer.HTTPServer):
	allow_reuse_address = True
	timeout = 120

	def handle_timeout(self):
		raise Exception('timeout')

class http_handler(SimpleHTTPServer.SimpleHTTPRequestHandler):

	def do_GET(s):

		#print s.requestline
		global chunked_encoding

		# if the request contains the hostname and port. strip it
		if s.path.startswith('http://') or s.path.startswith('https://'):
			s.path = s.path[8:]
			s.path = s.path[s.path.find('/'):]

		s.path = os.path.normpath(s.path)

		if s.path == '/redirect':
			s.send_response(301)
			s.send_header("Location", "/test_file")
			s.send_header("Connection", "close")
			s.end_headers()
		elif s.path == '/infinite_redirect':
			s.send_response(301)
			s.send_header("Location", "/infinite_redirect")
			s.send_header("Connection", "close")
			s.end_headers()
		elif s.path == '/relative/redirect':
			s.send_response(301)
			s.send_header("Location", "../test_file")
			s.send_header("Connection", "close")
			s.end_headers()
		elif s.path.startswith('/announce'):
			s.send_response(200)
			response = 'd8:intervali1800e8:completei1e10:incompletei1e5:peers0:e'
			s.send_header("Content-Length", "%d" % len(response))
			s.send_header("Connection", "close")
			s.end_headers()
			s.wfile.write(response)
		elif os.path.split(s.path)[1].startswith('seed?'):
			query = s.path[6:]
			args_raw = query.split('&')
			args = {}
			for a in args_raw:
				kvp = a.split('=')
				args[kvp[0]] = kvp[1]
			piece = int(args['piece'])
			ranges = args['ranges'].split('-')

			try:
				filename = s.path[1:s.path.find('seed?') + 4]
				#print 'filename = %s' % filename
				f = open(filename, 'rb')
				f.seek(piece * 64 * 1024 + int(ranges[0]))
				data = f.read(int(ranges[1]) - int(ranges[0]) + 1)
				f.close()

				s.send_response(200)
				print 'sending %d bytes' % len(data)
				s.send_header("Content-Length", "%d" % len(data))
				s.end_headers()
				s.wfile.write(data);
			except Exception, e:
				print 'FILE NOT FOUND: ', os.getcwd(), filename
				s.send_response(404)
				s.send_header("Content-Length", "0")
				s.end_headers()
		else:
			try:
				filename = s.path[1:]
				# serve file by invoking default handler
				f = open(filename, 'rb')
				size = int(os.stat(filename).st_size)
				start_range = 0
				end_range = size
				if 'Range' in s.headers:
					s.send_response(206)
					s.send_header('Content-Range', 'bytes ' + str(start_range) + '-' + str(end_range - 1) + '/' + str(size))
					st, e = s.headers['range'][6:].split('-', 1)
					sl = len(st)
					el = len(e)
					if sl > 0:
						start_range = int(st)
						if el > 0:
							end_range = int(e) + 1
					elif el > 0:
						ei = int(e)
						if ei < size:
							start_range = size - ei
				else:
					s.send_response(200)
				s.send_header('Accept-Ranges', 'bytes')
				if chunked_encoding:
					s.send_header('Transfer-Encoding', 'chunked')
				s.send_header('Content-Length', end_range - start_range)
				if filename.endswith('.gz'):
					s.send_header('Content-Encoding', 'gzip')
{'fraction_non_alphanumeric': 0.16518483855872718, 'fraction_numerical': 0.020589611605053813, 'mean_word_length': 3.353021045485404, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4358245', 'n_tokens_mistral': 2897, 'n_tokens_neox': 2343, 'n_words': 467}