<commit_before>import os
import select
import socket
import sys
import threading
import time

class config: None

config.max_threads = 8192
config.port = 7777
config.recognised_selectors = ['0', '1', '5', '9', 'g', 'h', 'I', 's']
config.request_max_size = 8192
config.socket_timeout = 1

# error(message)
# Print error message to stderr
def error(message):
	program_name = os.path.basename(sys.argv[0])
	print('%s: Error: %s' % (program_name, message), file = sys.stderr)

# die(message, status = 1) -> (Never returns)
# Print error message to stderr and exit with status code
def die(message, status = 1):
	error(message)
	sys.exit(status)

# bind(port, backlog = 1) → [sockets...]
# Binds to all available (TCP) interfaces on specified port and returns the sockets
# backlog controls how many connections allowed to wait handling before system drops new ones
def bind(port, backlog = 1):
	# Based on code in https://docs.python.org/3/library/socket.html
	sockets = []
	for res in socket.getaddrinfo(None, port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE):
		af, socktype, proto, canonname, sa = res
		
		try:
			s = socket.socket(af, socktype, proto)
		except OSError:
			continue
		# Make IPv6 socket only bind on IPv6 address, otherwise may clash with IPv4 and not get enabled
		if af == socket.AF_INET6:
			try:
				s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
			except OSError:
				pass
		try:
			s.bind(sa)
			s.listen(backlog)
		except OSError:
			s.close()
			continue

		sockets.append(s)
	
	return sockets


# drop_privileges()
# Drops set[ug]id, die()s if unsuccesful
def drop_privileges():
	try:
		uid = os.getuid()
		gid = os.getgid()
		os.setresgid(gid, gid, gid)
		os.setresuid(uid, uid, uid)
	except:
		die('Unable to drop privileges')

class Protocol:
	gopher, http = range(2)

class RequestError(Exception):
	def __init__(self, message):
		self.message = message
	def __str__(self):
		return 'Error with handling request: ' + self.message

# extract_selector_path(selector_path) -> selector, path
# Extract selector and path components from a HTTP path
def extract_selector_path(selector_path):
	if len(selector_path) > 0 and selector_path[0] == '/':
		selector_path = selector_path[1:]
	
	if len(selector_path) == 0: # / is by default of type 1
		selector = '1'
		path = selector_path
	elif selector_path[0] in config.recognised_selectors: # Requested path has a selector we recognise, extract it
		selector = selector_path[0]
		path = selector_path[1:]
	else: # We couldn't recognise any selector, return None for it
		selector = None
		path = selector_path
	
	return selector, path

# get_request(sock) -> path, protocol, rest
# Read request from socket and parse it.
# path is the requested path, protocol is Protocol.gopher or Protocol.http depending on the request protocol
# rest is protocol-dependant information
def get_request(sock):
	request = b''
	while True:
		try:
			data = sock.recv(1024)
		except socket.timeout:
			raise RequestError('Remote end timed out')
		if not data: # Other end hung up before sending a header
			raise RequestError('Remote end hung up unexpectedly')
		if len(request) >= config.request_max_size:
			raise RequestError('Request too long')

		request += data

		if b'\n' in request: # First line has been sent, all we care about for now
			break
	
	request = request.decode('utf-8')
	first_line = request.split('\n')[0]
	if first_line[-1] == '\r':
		first_line = first_line[:-1]
	first_line = first_line.split(' ')<commit_msg>Minor performance tweak<commit_after>import os
import select
import socket
import sys
import threading
import time

class config: None

config.max_threads = 8192
config.port = 7777
config.recognised_selectors = ['0', '1', '5', '9', 'g', 'h', 'I', 's']
config.request_max_size = 8192
config.socket_timeout = 1

# error(message)
# Print error message to stderr
def error(message):
	program_name = os.path.basename(sys.argv[0])
	print('%s: Error: %s' % (program_name, message), file = sys.stderr)

# die(message, status = 1) -> (Never returns)
# Print error message to stderr and exit with status code
def die(message, status = 1):
	error(message)
	sys.exit(status)

# bind(port, backlog = 1) → [sockets...]
# Binds to all available (TCP) interfaces on specified port and returns the sockets
# backlog controls how many connections allowed to wait handling before system drops new ones
def bind(port, backlog = 1):
	# Based on code in https://docs.python.org/3/library/socket.html
	sockets = []
	for res in socket.getaddrinfo(None, port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE):
		af, socktype, proto, canonname, sa = res
		
		try:
			s = socket.socket(af, socktype, proto)
		except OSError:
			continue
		# Make IPv6 socket only bind on IPv6 address, otherwise may clash with IPv4 and not get enabled
		if af == socket.AF_INET6:
			try:
				s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
			except OSError:
				pass
		try:
			s.bind(sa)
			s.listen(backlog)
		except OSError:
			s.close()
			continue

		sockets.append(s)
	
	return sockets


# drop_privileges()
# Drops set[ug]id, die()s if unsuccesful
def drop_privileges():
	try:
		uid = os.getuid()
		gid = os.getgid()
		os.setresgid(gid, gid, gid)
		os.setresuid(uid, uid, uid)
	except:
		die('Unable to drop privileges')

class Protocol:
	gopher, http = range(2)

class RequestError(Exception):
	def __init__(self, message):
		self.message = message
	def __str__(self):
		return 'Error with handling request: ' + self.message

# extract_selector_path(selector_path) -> selector, path
# Extract selector and path components from a HTTP path
def extract_selector_path(selector_path):
	if len(selector_path) > 0 and selector_path[0] == '/':
		selector_path = selector_path[1:]
	
	if len(selector_path) == 0: # / is by default of type 1
		selector = '1'
		path = selector_path
	elif selector_path[0] in config.recognised_selectors: # Requested path has a selector we recognise, extract it
		selector = selector_path[0]
		path = selector_path[1:]
	else: # We couldn't recognise any selector, return None for it
		selector = None
		path = selector_path
	
	return selector, path

# get_request(sock) -> path, protocol, rest
# Read request from socket and parse it.
# path is the requested path, protocol is Protocol.gopher or Protocol.http depending on the request protocol
# rest is protocol-dependant information
def get_request(sock):
	request = b''
	while True:
		try:
			data = sock.recv(1024)
		except socket.timeout:
			raise RequestError('Remote end timed out')
		if not data: # Other end hung up before sending a header
			raise RequestError('Remote end hung up unexpectedly')
		if len(request) >= config.request_max_size:
			raise RequestError('Request too long')

		request += data

		if b'\n' in data: # First line has been sent, all we care about for now
			break
	
	request = request.decode('utf-8')
	first_line = request.split('\n')[0]
	if first_line[-1] == '\r':
		first_line = first_line[:-1]
	first_line = first_line.split(' ')
{'fraction_non_alphanumeric': 0.10888415067519545, 'fraction_numerical': 0.014214641080312722, 'mean_word_length': 4.254667662434652, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '429738', 'n_tokens_mistral': 2527, 'n_tokens_neox': 2269, 'n_words': 768}