<commit_before>import logging

from pylons import request, response, session, tmpl_context as c, url
from pylons.controllers.util import abort, redirect

from chunkymonkey.lib.base import BaseController, render

from socket import *
import urllib
from urlparse import urlparse

log = logging.getLogger(__name__)

class ViewController(BaseController):

    def index(self):
    	q = request.params.get('q', None)
    	useragent = request.headers.get('User-Agent', None)
		
    	if q:
			url = urlparse(q)

			sock = socket(AF_INET, SOCK_STREAM)
			sock.connect( (url.hostname, ( url.port if url.port else 80) ) )
			httpRequest = 'GET %s HTTP/1.1\r\nHost: %s\r\nAccept: text/html\r\nUser-Agent: %s\r\n\r\n' % (url.path, url.hostname, useragent)
			
			log.info(httpRequest)
			
			# Force it to timeout so we can detect the end of the stream
			sock.settimeout(1)
			sock.send(httpRequest)
			
			# Read all content
			total_data=[]
			while True:
				try:
					data = sock.recv(4096)
				except:
					# Timeout exception
					break
				if not data: break
				total_data.append(data)
			sock.close()
<commit_msg>prevents mako from throwing an AttributeError when c.chunks is undefined<commit_after>import logging

from pylons import request, response, session, tmpl_context as c, url
from pylons.controllers.util import abort, redirect

from chunkymonkey.lib.base import BaseController, render

from socket import *
import urllib
from urlparse import urlparse

log = logging.getLogger(__name__)

class ViewController(BaseController):

    def index(self):
    	q = request.params.get('q', None)
    	useragent = request.headers.get('User-Agent', None)
        c.chunks = []
		
    	if q:
			url = urlparse(q)

			sock = socket(AF_INET, SOCK_STREAM)
			sock.connect( (url.hostname, ( url.port if url.port else 80) ) )
			httpRequest = 'GET %s HTTP/1.1\r\nHost: %s\r\nAccept: text/html\r\nUser-Agent: %s\r\n\r\n' % (url.path, url.hostname, useragent)
			
			log.info(httpRequest)
			
			# Force it to timeout so we can detect the end of the stream
			sock.settimeout(1)
			sock.send(httpRequest)
			
			# Read all content
			total_data=[]
			while True:
				try:
					data = sock.recv(4096)
				except:
					# Timeout exception
					break
				if not data: break
				total_data.append(data)
			sock.close()

{'fraction_non_alphanumeric': 0.10551454624402953, 'fraction_numerical': 0.007815892314372557, 'mean_word_length': 3.2275229357798163, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4646840', 'n_tokens_mistral': 902, 'n_tokens_neox': 771, 'n_words': 214}