<commit_before>"""\
This module contains the client based connections.

Blocking Example Usage:

>>> # Create the object and connect to the server
>>> c = netlib.Connection("127.0.0.1", 6329)
>>> if failed(c.connect()):
>>>    print "Could not connect to the server"
>>>    sys.exit(1)
>>>
>>> if failed(c.login("username", "password")):
>>> 	print "Could not login"
>>> 	sys.exit(1)
>>>
>>> c.disconnect()
>>>

Non-Blocking Example Usage:

>>> # Create the object and connect to the server
>>> c = netlib.Connection("127.0.0.1", 6329, nb=1)
>>>
>>> c.connect()
>>> c.login("username", "password")
>>>
>>> # Wait for the connection to be complete
>>> if failed(c.wait()):
>>>    print "Could not connect to the server"
>>>    sys.exit(1)
>>>
>>> r = c.poll()
>>> while r == None:
>>> 	r = c.poll()
>>>
>>> 	# Do some other stuff!
>>> 	pass
>>>
>>> if failed(r):
>>> 	print "Could not login"
>>> 	sys.exit(1)
>>>
>>> # Disconnect and cleanup
>>> c.disconnect()
>>>
"""

# Python Imports
import encodings.idna
import re
import socket
import types
import urllib

# Local imports
import xstruct
import objects
constants = objects.constants

from version import version
from common import Connection, l, SSLWrapper, _continue

def failed(object):
	if type(object) == types.TupleType:
		return failed(object[0])
	else:
		if isinstance(object, objects.Fail):
			return True
		if isinstance(object, bool):
			return not object
	return False

sequence_max = 4294967296

def url2bits(line):
	urlspliter = r'(.*?://)?(((.*):(.*)@)|(.*)@)?(.*?)(:(.*?))?(/.*?)?$'
	groups = re.compile(urlspliter, re.M).search(line).groups()
	
	proto = groups[0]

	if not groups[3] is None:
		username = groups[3]
	elif not groups[5] is None:
		username = groups[5]
	else:
		username = None

	server = groups[6]
	port = groups[8]

	password = groups[4]
	if not password is None:
		if password[-1] is '@':
			password = password[:-1]

	game = groups[9]
	if not game is None:
		game = urllib.unquote_plus(game)
		if game[0] == '/':
			game = game[1:]
		if len(game) == 0:
			game = None

	if proto is None:
		one = server
	else:
		one = "%s%s" % (proto, server)

	if not port is None:
		one = "%s:%s" % (one, port)

	return (one, username, game, password)


class ClientConnection(Connection):
	"""\
	Class for a connection from the client side.
	"""

	def __init__(self, host=None, port=None, nb=0, debug=False):
		Connection.__init__(self)

		self.buffered['undescribed'] = {}
		self.buffered['store'] = {}

		if host != None:
			self.setup(host, port, nb, debug)

		self.__desc = False

	def setup(self, host, port=None, nb=0, debug=False, proxy=None):
		"""\
		*Internal*

		Sets up the socket for a connection.
		"""
		hoststring = host
		self.proxy = None
		
		if hoststring.startswith("tphttp://") or hoststring.startswith("tphttps://"):
			hoststring = hoststring[2:]

		if hoststring.startswith("http://") or hoststring.startswith("https://"):
			import urllib
			opener = None

			# use enviroment varibles
			if proxy == None:
				opener = urllib.FancyURLopener()
			elif proxy == "":
				# Don't use any proxies
				opener = urllib.FancyURLopener({})
			else:
				if hoststring.startswith("http://"):
					opener = urlib.FancyURLopener({'http': proxy})
				elif hoststring.startswith("https://"):
					opener = urlib.FancyURLopener({'https': proxy})
				else:
					raise "URL Error..."

			import random, string
			url = "/"
			for i in range(0, 12):
				url += random.choice(string.letters+string.digits)

			o = opener.open(hoststring + url, "")
			s = socket.fromfd(o.fileno(), socket.AF_INET, socket.SOCK_STREAM)

##			# Read in the headers
##			buffer = ""
##			while not buffer.endswith("\r\n\r\n"):
##				print "buffer:", repr(buffer)
##				try:
##					buffer += s.recv(1)
##				except socket.error, e:
##					pass
##			print "Finished the http headers..."

		else:
			if hoststring.startswith("tp://") or hoststring.startswith("tps://"):
				if hoststring.startswith("tp://"):
					host = hoststring[5:]
					if not port:
						port = 6923
				elif hoststring.startswith("tps://"):
					host = hoststring[6:]
					if not port:
						port = 6924

				if host.count(":") > 0:
					host, port = host.split(':', 1)
					port = int(port)
			else:
				if hoststring.count(":") > 0:
					host, port = hoststring.split(':', 1)
					port = int(port)
				else:
					host = hoststring

					if not port:
						port = 6923

			print "Connecting to", host, type(host), port, type(port)

			s = None
			for af, socktype, proto, cannoname, sa in \
					socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):

				try:
					s = socket.socket(af, socktype, proto)
					if debug:
						print "Trying to connect to connect: (%s, %s)" % (host, port)

					s.connect(sa)
					break
				except socket.error, msg:
					if debug:
						print "Connect fail: (%s, %s)" % (host, port)
					if s:
						s.close()

					s = None
					continue

			if not s:
				raise socket.error, msg

			if hoststring.startswith("tps://"):
				print "Creating SSL wrapper...."
				s = SSLWrapper(s)

		self.hoststring = hoststring
		self.host = host
		self.port = port

		Connection.setup(self, s, nb=nb, debug=debug)
		self.no = 1

	def _description_error(self, p):
		# Need to figure out how to do non-blocking properly...
		#Send a request for the description
		if not self.__desc:
			q = objects.OrderDesc_Get(p.sequence-1, [p._subtype])
			self._send(q)

			self.__desc = True

		q = self._recv(p.sequence-1)
		if q != None and isinstance(q, objects.Sequence):
			q = self._recv(p.sequence-1)

		if q != None and isinstance(q, objects.Description):
			self.__desc = False

			# Register the desciption
			q.register()

	def _common(self):
		"""\
		*Internal*

		Does all the common goodness.
		"""
		# Increase the no number
		self.no += 1

	def _okfail(self, no):
		"""\
		*Internal*

		Completes the an ok or fail function.
		"""
		p = self._recv(no)
		if not p:
			return None

		# Check it's the reply we are after
		if isinstance(p, objects.OK):
			return True, p.s
		elif isinstance(p, objects.Fail):
			return False, p.s
		else:
			# We got a bad packet
			raise IOError("Bad Packet was received")

	def _get_single(self, type, no):
		"""\
		*Internal*

		Completes the function which only get a signle packet returned.
		"""
		p = self._recv(no)
		if not p:
			return None

		if isinstance(p, type):
			return p
		elif isinstance(p, objects.Fail):
			return False, p.s
		else:
			raise IOError("Bad Pakcet was received")

	def _get_header(self, type, no, callback=None):
		"""\
		*Internal*

		Completes the get_* function.
		"""
		p = self._recv(no)
		if not p:
			return None

		if isinstance(p, objects.Fail):
			# The whole command failed :(
			return (False, p.s)
#			return [(False, p.s)]
		elif isinstance(p, type):
			# Must only be one, so return
			if not callback is None:
				callback(p)

			return [p]
		elif not isinstance(p, objects.Sequence):
			# We got a bad packet
			raise IOError("Bad Packet was received %s" % p)

		# We have to wait on multiple packets
		self.buffered['store'][no] = []

		#print "Packets to get", p.number

		if self._noblock():
			# Do the commands in non-blocking mode
			self._next(self._get_finish, no)
			for i in range(0, p.number):
				self._next(self._get_data, type, no, callback)

			# Keep the polling going
			return _continue

		else:
			# Do the commands in blocking mode
			for i in range(0, p.number):
				self._get_data(type, no, callback)

			return self._get_finish(no)

	def _get_data(self, type, no, callback=None):
		"""\
		*Internal*

		Completes the get_* function.
		"""
		p = self._recv(no)

		if p != None:
			if not callback is None:
				callback(p)

			if isinstance(p, objects.Fail):
				p = (False, p.s)
			elif not isinstance(p, type):
				raise IOError("Bad Packet was received %s" % p)

			self.buffered['store'][no].append(p)

			if self._noblock():
				return _continue

	def _get_finish(self, no):
		"""\
		*Internal*

		Completes the get_* functions.
		"""
		store = self.buffered['store'][no]
		del self.buffered['store'][no]
		return l(store)

	def _get_ids(self, type, key, position, amount=-1, raw=False):
		"""\
		*Internal*

		Send a GetID packet and setup for a IDSequence result.
		"""
		self._common()

		p = type(self.no, key, position, amount)
		self._send(p)

		if self._noblock():
			self._append(self._get_idsequence, self.no, False, raw)
			return None
		else:
			return self._get_idsequence(self.no, False, raw)

	def _get_idsequence(self, no, iter=False, raw=False):
		"""\
		*Internal*

		Finishes any function which gets an IDSequence.
		"""
		p = self._recv(no)
		if not p:
			return None

		# Check it's the reply we are after
		if isinstance(p, objects.Fail):
			return False, p.s
		elif isinstance(p, objects.IDSequence):
			if iter:
				return p.iter()
			elif raw:
				return p
			else:
				return p.ids
		else:
			# We got a bad packet
			raise IOError("Bad Packet was received")

	class IDIter(object):
		"""\
		*Internal*

		Class used to iterate over an ID list. It will get more IDs as needed.

		On a non-blocking connection the IDIter will return (None, None) while
		no data is ready. This makes it good to use in an event loop.

		On a blocking connection the IDIter will wait till the information is
		ready.
		"""
		def __init__(self, connection, type, amount=30):
			"""\
			IDIter(connection, type, amount=10)

			connection
				Is a ClientConnection
			type
				Is the ID of the GetID packet to send
			amount
				Is the amount of IDs to get at one time
			"""
			self.connection = connection
			self.type = type
			self.amount = amount

			self.total = None

			self.key = None
			self.remaining = None

			self.position = None
			self.ids = None

			# Send out a packet if we are non-blocking
			if self.connection._noblock():
				self.next()

		def __iter__(self):
			return self

		def next(self):
			"""\
			Get the next (ids, modified time) pair.
			"""

			# Get the total number of IDs
			if self.key is None and self.remaining is None:
				if self.ids is None:
					self.ids = []
					p = self.connection._get_ids(self.type, -1, 0, 0, raw=True)
				else:
					p = self.connect.poll()

				# Check for Non-blocking mode
				if p is None:
					return (None, None)
				# Check for an error
				elif failed(p):
					raise IOError("Failed to get remaining IDs")

				if self.total == None:
					self.total = p.left

				self.remaining = p.left
				self.key = p.key
				self.position = 0

			# Get more IDs
			if len(self.ids) <= 0:
				no = self.remaining
				if no <= 0:
					raise StopIteration()
				elif no > self.amount:<commit_msg>Output the error reason too.<commit_after>"""\
This module contains the client based connections.

Blocking Example Usage:

>>> # Create the object and connect to the server
>>> c = netlib.Connection("127.0.0.1", 6329)
>>> if failed(c.connect()):
>>>    print "Could not connect to the server"
>>>    sys.exit(1)
>>>
>>> if failed(c.login("username", "password")):
>>> 	print "Could not login"
>>> 	sys.exit(1)
>>>
>>> c.disconnect()
>>>

Non-Blocking Example Usage:

>>> # Create the object and connect to the server
>>> c = netlib.Connection("127.0.0.1", 6329, nb=1)
>>>
>>> c.connect()
>>> c.login("username", "password")
>>>
>>> # Wait for the connection to be complete
>>> if failed(c.wait()):
>>>    print "Could not connect to the server"
>>>    sys.exit(1)
>>>
>>> r = c.poll()
>>> while r == None:
>>> 	r = c.poll()
>>>
>>> 	# Do some other stuff!
>>> 	pass
>>>
>>> if failed(r):
>>> 	print "Could not login"
>>> 	sys.exit(1)
>>>
>>> # Disconnect and cleanup
>>> c.disconnect()
>>>
"""

# Python Imports
import encodings.idna
import re
import socket
import types
import urllib

# Local imports
import xstruct
import objects
constants = objects.constants

from version import version
from common import Connection, l, SSLWrapper, _continue

def failed(object):
	if type(object) == types.TupleType:
		return failed(object[0])
	else:
		if isinstance(object, objects.Fail):
			return True
		if isinstance(object, bool):
			return not object
	return False

sequence_max = 4294967296

def url2bits(line):
	urlspliter = r'(.*?://)?(((.*):(.*)@)|(.*)@)?(.*?)(:(.*?))?(/.*?)?$'
	groups = re.compile(urlspliter, re.M).search(line).groups()
	
	proto = groups[0]

	if not groups[3] is None:
		username = groups[3]
	elif not groups[5] is None:
		username = groups[5]
	else:
		username = None

	server = groups[6]
	port = groups[8]

	password = groups[4]
	if not password is None:
		if password[-1] is '@':
			password = password[:-1]

	game = groups[9]
	if not game is None:
		game = urllib.unquote_plus(game)
		if game[0] == '/':
			game = game[1:]
		if len(game) == 0:
			game = None

	if proto is None:
		one = server
	else:
		one = "%s%s" % (proto, server)

	if not port is None:
		one = "%s:%s" % (one, port)

	return (one, username, game, password)


class ClientConnection(Connection):
	"""\
	Class for a connection from the client side.
	"""

	def __init__(self, host=None, port=None, nb=0, debug=False):
		Connection.__init__(self)

		self.buffered['undescribed'] = {}
		self.buffered['store'] = {}

		if host != None:
			self.setup(host, port, nb, debug)

		self.__desc = False

	def setup(self, host, port=None, nb=0, debug=False, proxy=None):
		"""\
		*Internal*

		Sets up the socket for a connection.
		"""
		hoststring = host
		self.proxy = None
		
		if hoststring.startswith("tphttp://") or hoststring.startswith("tphttps://"):
			hoststring = hoststring[2:]

		if hoststring.startswith("http://") or hoststring.startswith("https://"):
			import urllib
			opener = None

			# use enviroment varibles
			if proxy == None:
				opener = urllib.FancyURLopener()
			elif proxy == "":
				# Don't use any proxies
				opener = urllib.FancyURLopener({})
			else:
				if hoststring.startswith("http://"):
					opener = urlib.FancyURLopener({'http': proxy})
				elif hoststring.startswith("https://"):
					opener = urlib.FancyURLopener({'https': proxy})
				else:
					raise "URL Error..."

			import random, string
			url = "/"
			for i in range(0, 12):
				url += random.choice(string.letters+string.digits)

			o = opener.open(hoststring + url, "")
			s = socket.fromfd(o.fileno(), socket.AF_INET, socket.SOCK_STREAM)

##			# Read in the headers
##			buffer = ""
##			while not buffer.endswith("\r\n\r\n"):
##				print "buffer:", repr(buffer)
##				try:
##					buffer += s.recv(1)
##				except socket.error, e:
##					pass
##			print "Finished the http headers..."

		else:
			if hoststring.startswith("tp://") or hoststring.startswith("tps://"):
				if hoststring.startswith("tp://"):
					host = hoststring[5:]
					if not port:
						port = 6923
				elif hoststring.startswith("tps://"):
					host = hoststring[6:]
					if not port:
						port = 6924

				if host.count(":") > 0:
					host, port = host.split(':', 1)
					port = int(port)
			else:
				if hoststring.count(":") > 0:
					host, port = hoststring.split(':', 1)
					port = int(port)
				else:
					host = hoststring

					if not port:
						port = 6923

			print "Connecting to", host, type(host), port, type(port)

			s = None
			for af, socktype, proto, cannoname, sa in \
					socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):

				try:
					s = socket.socket(af, socktype, proto)
					if debug:
						print "Trying to connect to connect: (%s, %s)" % (host, port)

					s.connect(sa)
					break
				except socket.error, msg:
					if debug:
						print "Connect fail: (%s, %s)" % (host, port)
					if s:
						s.close()

					s = None
					continue

			if not s:
				raise socket.error, msg

			if hoststring.startswith("tps://"):
				print "Creating SSL wrapper...."
				s = SSLWrapper(s)

		self.hoststring = hoststring
		self.host = host
		self.port = port

		Connection.setup(self, s, nb=nb, debug=debug)
		self.no = 1

	def _description_error(self, p):
		# Need to figure out how to do non-blocking properly...
		#Send a request for the description
		if not self.__desc:
			q = objects.OrderDesc_Get(p.sequence-1, [p._subtype])
			self._send(q)

			self.__desc = True

		q = self._recv(p.sequence-1)
		if q != None and isinstance(q, objects.Sequence):
			q = self._recv(p.sequence-1)

		if q != None and isinstance(q, objects.Description):
			self.__desc = False

			# Register the desciption
			q.register()

	def _common(self):
		"""\
		*Internal*

		Does all the common goodness.
		"""
		# Increase the no number
		self.no += 1

	def _okfail(self, no):
		"""\
		*Internal*

		Completes the an ok or fail function.
		"""
		p = self._recv(no)
		if not p:
			return None

		# Check it's the reply we are after
		if isinstance(p, objects.OK):
			return True, p.s
		elif isinstance(p, objects.Fail):
			return False, p.s
		else:
			# We got a bad packet
			raise IOError("Bad Packet was received")

	def _get_single(self, type, no):
		"""\
		*Internal*

		Completes the function which only get a signle packet returned.
		"""
		p = self._recv(no)
		if not p:
			return None

		if isinstance(p, type):
			return p
		elif isinstance(p, objects.Fail):
			return False, p.s
		else:
			raise IOError("Bad Pakcet was received")

	def _get_header(self, type, no, callback=None):
		"""\
		*Internal*

		Completes the get_* function.
		"""
		p = self._recv(no)
		if not p:
			return None

		if isinstance(p, objects.Fail):
			# The whole command failed :(
			return (False, p.s)
#			return [(False, p.s)]
		elif isinstance(p, type):
			# Must only be one, so return
			if not callback is None:
				callback(p)

			return [p]
		elif not isinstance(p, objects.Sequence):
			# We got a bad packet
			raise IOError("Bad Packet was received %s" % p)

		# We have to wait on multiple packets
		self.buffered['store'][no] = []

		#print "Packets to get", p.number

		if self._noblock():
			# Do the commands in non-blocking mode
			self._next(self._get_finish, no)
			for i in range(0, p.number):
				self._next(self._get_data, type, no, callback)

			# Keep the polling going
			return _continue

		else:
			# Do the commands in blocking mode
			for i in range(0, p.number):
				self._get_data(type, no, callback)

			return self._get_finish(no)

	def _get_data(self, type, no, callback=None):
		"""\
		*Internal*

		Completes the get_* function.
		"""
		p = self._recv(no)

		if p != None:
			if not callback is None:
				callback(p)

			if isinstance(p, objects.Fail):
				p = (False, p.s)
			elif not isinstance(p, type):
				raise IOError("Bad Packet was received %s" % p)

			self.buffered['store'][no].append(p)

			if self._noblock():
				return _continue

	def _get_finish(self, no):
		"""\
		*Internal*

		Completes the get_* functions.
		"""
		store = self.buffered['store'][no]
		del self.buffered['store'][no]
		return l(store)

	def _get_ids(self, type, key, position, amount=-1, raw=False):
		"""\
		*Internal*

		Send a GetID packet and setup for a IDSequence result.
		"""
		self._common()

		p = type(self.no, key, position, amount)
		self._send(p)

		if self._noblock():
			self._append(self._get_idsequence, self.no, False, raw)
			return None
		else:
			return self._get_idsequence(self.no, False, raw)

	def _get_idsequence(self, no, iter=False, raw=False):
		"""\
		*Internal*

		Finishes any function which gets an IDSequence.
		"""
		p = self._recv(no)
		if not p:
			return None

		# Check it's the reply we are after
		if isinstance(p, objects.Fail):
			return False, p.s
		elif isinstance(p, objects.IDSequence):
			if iter:
				return p.iter()
			elif raw:
				return p
			else:
				return p.ids
		else:
			# We got a bad packet
			raise IOError("Bad Packet was received")

	class IDIter(object):
		"""\
		*Internal*

		Class used to iterate over an ID list. It will get more IDs as needed.

		On a non-blocking connection the IDIter will return (None, None) while
		no data is ready. This makes it good to use in an event loop.

		On a blocking connection the IDIter will wait till the information is
		ready.
		"""
		def __init__(self, connection, type, amount=30):
			"""\
			IDIter(connection, type, amount=10)

			connection
				Is a ClientConnection
			type
				Is the ID of the GetID packet to send
			amount
				Is the amount of IDs to get at one time
			"""
			self.connection = connection
			self.type = type
			self.amount = amount

			self.total = None

			self.key = None
			self.remaining = None

			self.position = None
			self.ids = None

			# Send out a packet if we are non-blocking
			if self.connection._noblock():
				self.next()

		def __iter__(self):
			return self

		def next(self):
			"""\
			Get the next (ids, modified time) pair.
			"""

			# Get the total number of IDs
			if self.key is None and self.remaining is None:
				if self.ids is None:
					self.ids = []
					p = self.connection._get_ids(self.type, -1, 0, 0, raw=True)
				else:
					p = self.connect.poll()

				# Check for Non-blocking mode
				if p is None:
					return (None, None)
				# Check for an error
				elif failed(p):
					raise IOError("Failed to get remaining IDs (%s)" % (p[1]))

				if self.total == None:
					self.total = p.left

				self.remaining = p.left
				self.key = p.key
				self.position = 0

			# Get more IDs
			if len(self.ids) <= 0:
				no = self.remaining
				if no <= 0:
					raise StopIteration()
				elif no > self.amount:
{'fraction_non_alphanumeric': 0.14293777338789332, 'fraction_numerical': 0.008983584967781384, 'mean_word_length': 2.9439807085883882, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 231, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3667776', 'n_tokens_mistral': 8809, 'n_tokens_neox': 7266, 'n_words': 2269}