<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
http.py

Created by <NAME> on 2011-12-02.
Copyright (c) 2011 Exa Networks. All rights reserved.
"""

import sys
import re
import socket
import errno

from .logger import Logger
logger = Logger()

from .version import version
from . import html

DEFAULT_READ_BUFFER_SIZE=4096

class regex:
	# XXX: Should this regex contain TRACE/CONNECT ?
	destination = re.compile("(GET|POST|PUT|HEAD|DELETE|OPTIONS|TRACE|CONNECT) (http://[^/]*|)(/[^ \r]*)(.*HTTP/.*\r?\nHost: ?)([^\r]*)(|\r?\n)", re.IGNORECASE)

	# Should we simply eat everything between : and \n to accept IPv6 address 
	x_forwarded_for = re.compile("(|\n)X-Forwarded-For: ?(((1?\d?\d)|(2([0-4]\d|5[0-5])))\.)(((1?\d?\d)|(2([0-4]\d|5[0-5])))\.)(((1?\d?\d)|(2([0-4]\d|5[0-5])))\.)((2([0-4]\d|5[0-5]))|(1?\d?\d))", re.IGNORECASE)

	connection = re.compile("\nConnection\s*:\s*([^\r\n]*)\s*\r?\n", re.IGNORECASE)

def _http (message):
	return """\
HTTP/1.1 200 OK
Date: Fri, 02 Dec 2011 09:29:44 GMT
Server: exaproxy/""" + str(version) + """ ("""+  sys.platform +""")
Content-Length: %d
Connection: close
Content-Type: text/html
Cache-control: private
Pragma: no-cache

%s""" % (len(message),message)


class HTTPFetcher (object):
	def __init__  (self,cid,host,port,request):
		self.io = None
		self._request = request
		self.cid = cid
		self.host = host
		self.port = port
		self._recv = self._fetch()

	def fileno (self):
		return self.io.fileno()

	def connect (self):
		logger.debug('connecting to webserver %s:%d' % (self.host,self.port), 'http %d' %self.cid)
		try:
			self.io = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
		except socket.error,e:
			logger.debug('problem create a connection to %s:%d' % (self.host,self.port), 'http %d' %self.cid)
			return False
		try:
			self.io.setblocking(0)
			self.io.connect((self.host, self.port))
			return True
		except socket.error,e:
			if e.errno in (errno.EINPROGRESS,):
				print "errno.EINPROGRESS -----------------------------------", self.host, self.port
				return True
			logger.debug('problem create a connection to %s:%d' % (self.host,self.port), 'http %d' %self.cid)
			self.close()
			return False

	def request (self):
		try:
			logger.debug('sending request to the website','http %d' % self.cid)
			number = self.io.send(self._request)
			logger.debug('sent %d bytes' % number,'http %d' % self.cid)
			self._request = self._request[number:]
			if not self._request:
				return True
			return False
		except socket.error,e:
#			if e.errno == errno.EISCONN:
#				break
#			if e.errno in (errno.EINPROGRESS,errno.EALREADY):
#				yield False
#				continue<commit_msg>remove lefover print<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
http.py

Created by <NAME> on 2011-12-02.
Copyright (c) 2011 Exa Networks. All rights reserved.
"""

import sys
import re
import socket
import errno

from .logger import Logger
logger = Logger()

from .version import version
from . import html

DEFAULT_READ_BUFFER_SIZE=4096

class regex:
	# XXX: Should this regex contain TRACE/CONNECT ?
	destination = re.compile("(GET|POST|PUT|HEAD|DELETE|OPTIONS|TRACE|CONNECT) (http://[^/]*|)(/[^ \r]*)(.*HTTP/.*\r?\nHost: ?)([^\r]*)(|\r?\n)", re.IGNORECASE)

	# Should we simply eat everything between : and \n to accept IPv6 address 
	x_forwarded_for = re.compile("(|\n)X-Forwarded-For: ?(((1?\d?\d)|(2([0-4]\d|5[0-5])))\.)(((1?\d?\d)|(2([0-4]\d|5[0-5])))\.)(((1?\d?\d)|(2([0-4]\d|5[0-5])))\.)((2([0-4]\d|5[0-5]))|(1?\d?\d))", re.IGNORECASE)

	connection = re.compile("\nConnection\s*:\s*([^\r\n]*)\s*\r?\n", re.IGNORECASE)

def _http (message):
	return """\
HTTP/1.1 200 OK
Date: Fri, 02 Dec 2011 09:29:44 GMT
Server: exaproxy/""" + str(version) + """ ("""+  sys.platform +""")
Content-Length: %d
Connection: close
Content-Type: text/html
Cache-control: private
Pragma: no-cache

%s""" % (len(message),message)


class HTTPFetcher (object):
	def __init__  (self,cid,host,port,request):
		self.io = None
		self._request = request
		self.cid = cid
		self.host = host
		self.port = port
		self._recv = self._fetch()

	def fileno (self):
		return self.io.fileno()

	def connect (self):
		logger.debug('connecting to webserver %s:%d' % (self.host,self.port), 'http %d' %self.cid)
		try:
			self.io = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
		except socket.error,e:
			logger.debug('problem create a connection to %s:%d' % (self.host,self.port), 'http %d' %self.cid)
			return False
		try:
			self.io.setblocking(0)
			self.io.connect((self.host, self.port))
			return True
		except socket.error,e:
			if e.errno in (errno.EINPROGRESS,):
				return True
			logger.debug('problem create a connection to %s:%d' % (self.host,self.port), 'http %d' %self.cid)
			self.close()
			return False

	def request (self):
		try:
			logger.debug('sending request to the website','http %d' % self.cid)
			number = self.io.send(self._request)
			logger.debug('sent %d bytes' % number,'http %d' % self.cid)
			self._request = self._request[number:]
			if not self._request:
				return True
			return False
		except socket.error,e:
#			if e.errno == errno.EISCONN:
#				break
#			if e.errno in (errno.EINPROGRESS,errno.EALREADY):
#				yield False
#				continue
{'fraction_non_alphanumeric': 0.19691487335745572, 'fraction_numerical': 0.02437630927442392, 'mean_word_length': 3.80073126142596, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2317028', 'n_tokens_mistral': 2222, 'n_tokens_neox': 2016, 'n_words': 453}