<commit_before>#!/usr/bin/python

class UDPLoggerStatistic:
	def __init__(self):
		self.results = {}

	def __str__(self):
		s = '%s\n' % str(self.__class__)
		for unix_timestamp in self.results:
			s += '\t%d\n' % unix_timestamp
			for i in self.results[unix_timestamp]:
				s += '\t\t%-25s\t%s\n' % (str(i), str(self.results[unix_timestamp][i]))
		return s

class ContentTypeStatistic(UDPLoggerStatistic):
	def save(self, database_connection):
		cursor = database_connection.cursor()
		cursor.execute('''CREATE TABLE IF NOT EXISTS content_type_statistics (
									timestamp INTEGER,
									content_type TEXT,
									count INTEGER,
									transferred INTEGER,
									PRIMARY KEY (timestamp, content_type)
								);''')
		for unix_timestamp in self.results:
			for content_type in self.results[unix_timestamp]:
				cursor.execute('''INSERT OR IGNORE INTO content_type_statistics (
											timestamp,
											content_type,
											count,
											transferred
										) VALUES (
											?,
											?,
											0,
											0
										);''', (unix_timestamp, content_type))
				cursor.execute('''UPDATE content_type_statistics
										SET
											count = count + ?,
											transferred = transferred + ?
										WHERE
											timestamp = ? AND
											content_type = ?;''', (self.results[unix_timestamp][content_type]['count'], self.results[unix_timestamp][content_type]['transferred'], unix_timestamp, content_type))
		database_connection.commit()

	def update(self, log):
		i = log.content_type.find(';')
		if i > -1:
			content_type = log.content_type[:i]
		else:
			content_type = log.content_type
		if not log.unix_timestamp in self.results:
			self.results[log.unix_timestamp] = {}
		if not content_type in self.results[log.unix_timestamp]:
			self.results[log.unix_timestamp][content_type] = {}
			self.results[log.unix_timestamp][content_type]['count'] = 0
			self.results[log.unix_timestamp][content_type]['transferred'] = 0
		self.results[log.unix_timestamp][content_type]['count'] += 1
		self.results[log.unix_timestamp][content_type]['transferred'] += log.bytes_outgoing

class HitStatistic(UDPLoggerStatistic):
	def save(self, database_connection):
		cursor = database_connection.cursor()
		cursor.execute('''CREATE TABLE IF NOT EXISTS hit_statistics (
									timestamp INTEGER,
									hits INTEGER,
									bytes_incoming INTEGER,
									bytes_outgoing INTEGER,
									PRIMARY KEY (timestamp)
								);''')
		for unix_timestamp in self.results:
			cursor.execute('''INSERT OR IGNORE INTO hit_statistics (
										timestamp,
										hits,
										bytes_incoming,
										bytes_outgoing
									) VALUES (
										?,
										0,
										0,
										0
									);''', (unix_timestamp,))
			cursor.execute('''UPDATE hit_statistics
									SET
										hits = hits + ?,
										bytes_incoming = bytes_incoming + ?,
										bytes_outgoing = bytes_outgoing + ?
									WHERE
										timestamp = ?;''', (self.results[unix_timestamp]['hits'], self.results[unix_timestamp]['bytes_incoming'], self.results[unix_timestamp]['bytes_outgoing'], unix_timestamp))
		database_connection.commit()

	def update(self, log):
		if not log.unix_timestamp in self.results:
			self.results[log.unix_timestamp] = {}
			self.results[log.unix_timestamp]['bytes_incoming'] = 0
			self.results[log.unix_timestamp]['bytes_outgoing'] = 0
			self.results[log.unix_timestamp]['hits'] = 0
		self.results[log.unix_timestamp]['bytes_incoming'] += log.bytes_incoming
		self.results[log.unix_timestamp]['bytes_outgoing'] += log.bytes_outgoing
		self.results[log.unix_timestamp]['hits'] += 1

class HostStatistic(UDPLoggerStatistic):
	def save(self, database_connection):
		cursor = database_connection.cursor()
		cursor.execute('''CREATE TABLE IF NOT EXISTS host_statistics (
									timestamp INTEGER,
									host TEXT,
									count INTEGER,
									PRIMARY KEY (timestamp, host)
								);''')
		for unix_timestamp in self.results:
			for host in self.results[unix_timestamp]:
				cursor.execute('''INSERT OR IGNORE INTO host_statistics (
											timestamp,
											host,
											count
										) VALUES (
											?,
											?,
											0
										);''', (unix_timestamp, host))
				cursor.execute('''UPDATE host_statistics
										SET
											count = count + ?
										WHERE
											timestamp = ? AND
											host = ?;''', (self.results[unix_timestamp][host], unix_timestamp, host))
		database_connection.commit()

	def update(self, log):
		i = log.host.find(':')
		if i > -1:
			host = log.host[:i]
		else:
			host = log.host
		if not log.unix_timestamp in self.results:<commit_msg>* Handle the case where a log-line has a null host header (i.e. request for   an IP address?)<commit_after>#!/usr/bin/python

class UDPLoggerStatistic:
	def __init__(self):
		self.results = {}

	def __str__(self):
		s = '%s\n' % str(self.__class__)
		for unix_timestamp in self.results:
			s += '\t%d\n' % unix_timestamp
			for i in self.results[unix_timestamp]:
				s += '\t\t%-25s\t%s\n' % (str(i), str(self.results[unix_timestamp][i]))
		return s

class ContentTypeStatistic(UDPLoggerStatistic):
	def save(self, database_connection):
		cursor = database_connection.cursor()
		cursor.execute('''CREATE TABLE IF NOT EXISTS content_type_statistics (
									timestamp INTEGER,
									content_type TEXT,
									count INTEGER,
									transferred INTEGER,
									PRIMARY KEY (timestamp, content_type)
								);''')
		for unix_timestamp in self.results:
			for content_type in self.results[unix_timestamp]:
				cursor.execute('''INSERT OR IGNORE INTO content_type_statistics (
											timestamp,
											content_type,
											count,
											transferred
										) VALUES (
											?,
											?,
											0,
											0
										);''', (unix_timestamp, content_type))
				cursor.execute('''UPDATE content_type_statistics
										SET
											count = count + ?,
											transferred = transferred + ?
										WHERE
											timestamp = ? AND
											content_type = ?;''', (self.results[unix_timestamp][content_type]['count'], self.results[unix_timestamp][content_type]['transferred'], unix_timestamp, content_type))
		database_connection.commit()

	def update(self, log):
		i = log.content_type.find(';')
		if i > -1:
			content_type = log.content_type[:i]
		else:
			content_type = log.content_type
		if not log.unix_timestamp in self.results:
			self.results[log.unix_timestamp] = {}
		if not content_type in self.results[log.unix_timestamp]:
			self.results[log.unix_timestamp][content_type] = {}
			self.results[log.unix_timestamp][content_type]['count'] = 0
			self.results[log.unix_timestamp][content_type]['transferred'] = 0
		self.results[log.unix_timestamp][content_type]['count'] += 1
		self.results[log.unix_timestamp][content_type]['transferred'] += log.bytes_outgoing

class HitStatistic(UDPLoggerStatistic):
	def save(self, database_connection):
		cursor = database_connection.cursor()
		cursor.execute('''CREATE TABLE IF NOT EXISTS hit_statistics (
									timestamp INTEGER,
									hits INTEGER,
									bytes_incoming INTEGER,
									bytes_outgoing INTEGER,
									PRIMARY KEY (timestamp)
								);''')
		for unix_timestamp in self.results:
			cursor.execute('''INSERT OR IGNORE INTO hit_statistics (
										timestamp,
										hits,
										bytes_incoming,
										bytes_outgoing
									) VALUES (
										?,
										0,
										0,
										0
									);''', (unix_timestamp,))
			cursor.execute('''UPDATE hit_statistics
									SET
										hits = hits + ?,
										bytes_incoming = bytes_incoming + ?,
										bytes_outgoing = bytes_outgoing + ?
									WHERE
										timestamp = ?;''', (self.results[unix_timestamp]['hits'], self.results[unix_timestamp]['bytes_incoming'], self.results[unix_timestamp]['bytes_outgoing'], unix_timestamp))
		database_connection.commit()

	def update(self, log):
		if not log.unix_timestamp in self.results:
			self.results[log.unix_timestamp] = {}
			self.results[log.unix_timestamp]['bytes_incoming'] = 0
			self.results[log.unix_timestamp]['bytes_outgoing'] = 0
			self.results[log.unix_timestamp]['hits'] = 0
		self.results[log.unix_timestamp]['bytes_incoming'] += log.bytes_incoming
		self.results[log.unix_timestamp]['bytes_outgoing'] += log.bytes_outgoing
		self.results[log.unix_timestamp]['hits'] += 1

class HostStatistic(UDPLoggerStatistic):
	def save(self, database_connection):
		cursor = database_connection.cursor()
		cursor.execute('''CREATE TABLE IF NOT EXISTS host_statistics (
									timestamp INTEGER,
									host TEXT,
									count INTEGER,
									PRIMARY KEY (timestamp, host)
								);''')
		for unix_timestamp in self.results:
			for host in self.results[unix_timestamp]:
				cursor.execute('''INSERT OR IGNORE INTO host_statistics (
											timestamp,
											host,
											count
										) VALUES (
											?,
											?,
											0
										);''', (unix_timestamp, host))
				cursor.execute('''UPDATE host_statistics
										SET
											count = count + ?
										WHERE
											timestamp = ? AND
											host = ?;''', (self.results[unix_timestamp][host], unix_timestamp, host))
		database_connection.commit()

	def update(self, log):
		if log.host is None:
			host = ""
		else:
			i = log.host.find(':')
			if i > -1:
				host = log.host[:i]
			else:
				host = log.host
		if not log.unix_timestamp in self.results:
{'fraction_non_alphanumeric': 0.1209277807063785, 'fraction_numerical': 0.003584607274644175, 'mean_word_length': 2.818840579710145, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5147174', 'n_tokens_mistral': 4249, 'n_tokens_neox': 2917, 'n_words': 576}