<commit_before>#!/usr/bin/env python3

import argparse
import datetime
import gzip
import os
import random
import sys

import inspect
_currentFile = os.path.abspath(inspect.getfile(inspect.currentframe()))
_currentDir = os.path.dirname(_currentFile)
_parentDir = os.path.dirname(os.path.dirname(_currentDir))
sys.path.insert(0, _parentDir)

from NanoPcap import Listener, Parser

class PcapFilterListener(Listener.PcapListener):

	def __init__(self, arguments):
		self._arguments = arguments

		self._lastPacketDatas = []
		self._outputFileName = None
		self._outputFile = None
		self._header = None

	def onPcapHeader(self, header):
		self._header = header

		#Warn on a larger snap len than before
		if self._arguments.snaplen > header.snaplen():
			print('WARNING: New snaplen is greater than original: %d > %d' % (self._arguments.snaplen, header.snaplen()))

		#Check if the link type is correct
		if self._arguments.required_link_type is not None and self._arguments.required_link_type != header.network():
			print('ERROR: Link type is %d instead of %d' % (header.network(), self._arguments.required_link_type))
			sys.exit(1)

		#Update with new snaplen
		snaplen = min(self._arguments.snaplen, header.snaplen())
		self._header.setSnaplen(snaplen)

		#Update with the new link link
		if self._arguments.link_type is not None:
			self._header.setNetwork(self._arguments.link_type)

	def onPcapRecord(self, recordHeader, data):
		if self._arguments.no_records:
			return

		#Filter

		#Check the time
		epochTime = recordHeader.epochNanos()
		if self._arguments.start is not None and epochTime < self._arguments.start:
			return
		if self._arguments.end is not None and epochTime > self._arguments.end:
			return

		#Drop?
		if self._arguments.drop_fraction > 0 and random.random() < self._arguments.drop_fraction:
			return

		#De-duplicate
		if self._arguments.deduplication_window > 0:
			found = False
			for packetData in self._lastPacketDatas:
				if packetData == data:
					found = True
					break

			self._lastPacketDatas.append(data)
			if len(self._lastPacketDatas) > self._arguments.deduplication_window:
				self._lastPacketDatas.pop(0)
			if found:
				return

		#Roll the file if necessary
		newOutputFileName = recordHeader.timestampDatetime().strftime(self._arguments.output)
		if newOutputFileName != self._outputFileName:
			if self._outputFile is not None:
				self._outputFile.close()

			#Write to the output file
			self._outputFileName = newOutputFileName
			directory, fileName = os.path.split(self._outputFileName)
			if directory != '' and not os.path.exists(directory):
				os.makedirs(directory)

			self._outputFile = gzip.open(self._outputFileName, 'wb') if self._outputFileName.endswith('.gz') else open(self._outputFileName, 'ab' if self._arguments.append else 'wb')
			#TODO: should this write the header in append mode if there is nothing yet in the file?
			if not (self._arguments.no_header or self._arguments.append):
				self._header.writeToFile(self._outputFile)

		#Update with new snaplen
		start = self._arguments.data_offset
		end = min(start + self._arguments.snaplen, len(data) - start - self._arguments.data_end_offset)
		truncatedData = data[start:end]
		recordHeader.setIncludedLength(len(truncatedData))
		if self._arguments.link_type is not None:
			recordHeader.setOriginalLength(len(truncatedData))

		#Update with the new timestamp if necessary
		if self._arguments.time_shift_seconds is not None:
			recordHeader.setTsSec(recordHeader.tsSec() + self._arguments.time_shift_seconds)

		#Write the header and data
		recordHeader.writeToFile(self._outputFile)<commit_msg>Filter - fix bug that caused append mode to not be applied to gzipped output files<commit_after>#!/usr/bin/env python3

import argparse
import datetime
import gzip
import os
import random
import sys

import inspect
_currentFile = os.path.abspath(inspect.getfile(inspect.currentframe()))
_currentDir = os.path.dirname(_currentFile)
_parentDir = os.path.dirname(os.path.dirname(_currentDir))
sys.path.insert(0, _parentDir)

from NanoPcap import Listener, Parser

class PcapFilterListener(Listener.PcapListener):

	def __init__(self, arguments):
		self._arguments = arguments

		self._lastPacketDatas = []
		self._outputFileName = None
		self._outputFile = None
		self._header = None

	def onPcapHeader(self, header):
		self._header = header

		#Warn on a larger snap len than before
		if self._arguments.snaplen > header.snaplen():
			print('WARNING: New snaplen is greater than original: %d > %d' % (self._arguments.snaplen, header.snaplen()))

		#Check if the link type is correct
		if self._arguments.required_link_type is not None and self._arguments.required_link_type != header.network():
			print('ERROR: Link type is %d instead of %d' % (header.network(), self._arguments.required_link_type))
			sys.exit(1)

		#Update with new snaplen
		snaplen = min(self._arguments.snaplen, header.snaplen())
		self._header.setSnaplen(snaplen)

		#Update with the new link link
		if self._arguments.link_type is not None:
			self._header.setNetwork(self._arguments.link_type)

	def onPcapRecord(self, recordHeader, data):
		if self._arguments.no_records:
			return

		#Filter

		#Check the time
		epochTime = recordHeader.epochNanos()
		if self._arguments.start is not None and epochTime < self._arguments.start:
			return
		if self._arguments.end is not None and epochTime > self._arguments.end:
			return

		#Drop?
		if self._arguments.drop_fraction > 0 and random.random() < self._arguments.drop_fraction:
			return

		#De-duplicate
		if self._arguments.deduplication_window > 0:
			found = False
			for packetData in self._lastPacketDatas:
				if packetData == data:
					found = True
					break

			self._lastPacketDatas.append(data)
			if len(self._lastPacketDatas) > self._arguments.deduplication_window:
				self._lastPacketDatas.pop(0)
			if found:
				return

		#Roll the file if necessary
		newOutputFileName = recordHeader.timestampDatetime().strftime(self._arguments.output)
		if newOutputFileName != self._outputFileName:
			if self._outputFile is not None:
				self._outputFile.close()

			#Write to the output file
			self._outputFileName = newOutputFileName
			directory, fileName = os.path.split(self._outputFileName)
			if directory != '' and not os.path.exists(directory):
				os.makedirs(directory)

			mode = 'ab' if self._arguments.append else 'wb'
			self._outputFile = gzip.open(self._outputFileName, mode) if self._outputFileName.endswith('.gz') else open(self._outputFileName, mode)
			#TODO: should this write the header in append mode if there is nothing yet in the file?
			if not (self._arguments.no_header or self._arguments.append):
				self._header.writeToFile(self._outputFile)

		#Update with new snaplen
		start = self._arguments.data_offset
		end = min(start + self._arguments.snaplen, len(data) - start - self._arguments.data_end_offset)
		truncatedData = data[start:end]
		recordHeader.setIncludedLength(len(truncatedData))
		if self._arguments.link_type is not None:
			recordHeader.setOriginalLength(len(truncatedData))

		#Update with the new timestamp if necessary
		if self._arguments.time_shift_seconds is not None:
			recordHeader.setTsSec(recordHeader.tsSec() + self._arguments.time_shift_seconds)

		#Write the header and data
		recordHeader.writeToFile(self._outputFile)
{'fraction_non_alphanumeric': 0.09455040871934604, 'fraction_numerical': 0.0016348773841961854, 'mean_word_length': 4.4906507105459985, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1745364', 'n_tokens_mistral': 2487, 'n_tokens_neox': 2259, 'n_words': 589}