<commit_before>import feedparser

def escaping(str):
	return str.replace('\\','\\\\').replace('"','\\"').replace('\n','')
	
def get(url, id, etag, modified):
	data = feedparser.parse(url, etag=etag, modified=modified)
	print '{'
	print ('"title": "%s",'% (data.feed.get('title', 'null'))).encode('utf-8')
	print ('"description": "%s",'% (data.feed.get('description', 'null'))).encode('utf-8')
	print ('"language": "%s",'% (data.feed.get('language', 'en'))).encode('utf-8')
	print ('"status": "%s",'% (data.status)).encode('utf-8')
	print ('"standard": "%s",'% (data.version)).encode('utf-8')
	print ('"etag": "%s",'% (data.get('etag', 'null'))).encode('utf-8')
	print ('"modified": "%s",'% (data.get('modified', 'null'))).encode('utf-8')
	if data.status == 301:
		print ('"moved": "%s",' % (data.href)).encode('utf-8')
	print ('"updated": "%s",'% (data.feed.get('published', 'null'))).encode('utf-8')
	print '"articles": ['.encode('utf-8')
	size = len(data.entries)
	counter = 0
	for article in data.entries:
		counter = counter + 1
		print ('{ "title": "%s",' % (article.get('title', 'null').replace('"','\\"'))).encode('utf-8')<commit_msg>Added HTTP header referrer and agent to the Feeder<commit_after>import feedparser

def escaping(str):
	return str.replace('\\','\\\\').replace('"','\\"').replace('\n','')
	
def get(url, id, etag, modified):
	data = feedparser.parse(url, etag=etag, modified=modified, agent='The Pieuvre/1.0 +http://www.thepieuvre.com/', referrer='http://www.thepieuvre.com/')
	print '{'
	print ('"title": "%s",'% (data.feed.get('title', 'null'))).encode('utf-8')
	print ('"description": "%s",'% (data.feed.get('description', 'null'))).encode('utf-8')
	print ('"language": "%s",'% (data.feed.get('language', 'en'))).encode('utf-8')
	print ('"status": "%s",'% (data.status)).encode('utf-8')
	print ('"standard": "%s",'% (data.version)).encode('utf-8')
	print ('"etag": "%s",'% (data.get('etag', 'null'))).encode('utf-8')
	print ('"modified": "%s",'% (data.get('modified', 'null'))).encode('utf-8')
	if data.status == 301:
		print ('"moved": "%s",' % (data.href)).encode('utf-8')
	print ('"updated": "%s",'% (data.feed.get('published', 'null'))).encode('utf-8')
	print '"articles": ['.encode('utf-8')
	size = len(data.entries)
	counter = 0
	for article in data.entries:
		counter = counter + 1
		print ('{ "title": "%s",' % (article.get('title', 'null').replace('"','\\"'))).encode('utf-8')
{'fraction_non_alphanumeric': 0.30734135213604313, 'fraction_numerical': 0.014102032351721278, 'mean_word_length': 5.644628099173554, 'pattern_counts': {'":': 22, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3499512', 'n_tokens_mistral': 926, 'n_tokens_neox': 899, 'n_words': 171}