<commit_before>import re, hashlib, pprint, socket, urllib, time
import urllib.request
from queue import Queue
from collections import Counter, deque, defaultdict
from html.parser import HTMLParser
from classes.fingerprints import Fingerprints
#from classes.requester2 import Requester
from classes.matcher import Match
from classes.request2 import Response, Requester
from classes.printer import Printer

class DiscoverAllCMS(object):
	# match all fingerprints against all responses
	# this might generate false positives

	def __init__(self, data):
		self.cache = data['cache']
		self.results = data['results']
		self.matcher = data['matcher']
		self.fps = data['fingerprints']
		self.printer = data['printer']

		# only used for pretty printing of debugging info
		self.tmp_set = set()

	def run(self):
		self.printer.print('Checking for more matches in cache (option -a)  ...', 1)

		# find matches for all the responses in the cache
		for fp_category in ['cms', 'platform']:
			for fp_type in self.fps.data[fp_category]:
				fps = self.fps.data[fp_category][fp_type]['fps']

				for response in self.cache.get_responses():
					matches = self.matcher.get_result(fps, response)
					for fp in matches:
						self.results.add( fp_category, fp['name'], fp['output'], fp )

						if (fp['name'], fp['output']) not in self.tmp_set:
							self.printer.print('- Found match: %s %s' % (fp['name'], fp['output']) , 2)

						self.tmp_set.add((fp['name'], fp['output']))


class DiscoverCMS(object):

	def __init__(self, options, data):
		self.printer = data['printer']
		self.matcher = data['matcher']
		self.requester = data['requester']
		self.result = data['results']
		self.printer = data['printer']

		self.batch_size = options['batch_size']
		self.num_cms_to_find = options['stop_after']
		self.find_all_cms = options['run_all']

		# only used for pretty printing of debugging info
		self.tmp_set = set()

		self.queue = defaultdict(list)
		for fp_type in data['fingerprints'].data['cms']:
			for fp in data['fingerprints'].data['cms'][fp_type]['fps']:
				self.queue[fp['url']].append(fp)



	def get_queue(self, cms=None):
		queue = []
		if cms is None:
			for i in range(self.batch_size):
				try:
					url, fp_list = self.queue.popitem()
					queue.append(fp_list)
				except KeyError:
					break
		else:
			# the following procedure is *not* optimal
			# the self.queue dict is completely destroyed and 
			# and rebuilt each time this procedure is called :(

			# create a temp queue dict
			tmp_queue = defaultdict(list)
			
			# remove elements from the dict until it is empty
			while len(self.queue) > 0:
				url, fp_list = self.queue.popitem()

				# remove all the elements of a queue entry's list
				# one-by-one and check if the fingerprints are
				# belong to the specified 'cms'
				tmp_list = []
				out_list = []
				
				while len(fp_list) > 0:
					# remove the fingerprint
					fp = fp_list.pop()

					# if the fingerprint matches the cms, add it to the 
					# out_list for the current url
					# otherwise add it to the tmp_list
					if fp['name'] == cms: 
						out_list.append(fp)
					else:
						tmp_list.append(fp)

				# if there are elements in tmp_list (the new list of fps that
				# that do *not* match the 'cms'), add it to the tmp_queue's entry
				# for the current url
				if len(tmp_list) > 0:
					tmp_queue[url].extend(tmp_list)

				# if matches for the specified cms have been found, add the list
				# to the fingerprintQueue for the requester
				if len(out_list) > 0:
					queue.append(out_list)

			# replace the queue with the tmp queue
			self.queue = tmp_queue

		return queue



	def run(self):
		batch_no = 0
		self.printer.print('Determining CMS type ...', 1)

		detected_cms = []
		stop_searching = len(detected_cms) >= self.num_cms_to_find

		while (not stop_searching or self.find_all_cms) and (not len(self.queue) == 0):
			self.printer.print('Checking fingerprint group no. %s ...' % (batch_no, ) , 3)

			# set the requester queue
			results = self.requester.run('CMS', self.get_queue())
			
			# search for CMS matches
			cms_matches = []
			while not results.empty():
				fingerprints, response = results.get()
				matches = self.matcher.get_result(fingerprints, response)
				if matches:
					cms_matches.extend([cms['name'] for cms in matches])

		
			# search for the found CMS versions
			for cms in cms_matches:

				# skip checking the cms, if it has already been detected
				if cms in detected_cms: continue<commit_msg>Fixed issue where the cms was not displayed in the output. This issue happend when the only one fingerprint for cms matches.<commit_after>import re, hashlib, pprint, socket, urllib, time
import urllib.request
from queue import Queue
from collections import Counter, deque, defaultdict
from html.parser import HTMLParser
from classes.fingerprints import Fingerprints
#from classes.requester2 import Requester
from classes.matcher import Match
from classes.request2 import Response, Requester
from classes.printer import Printer

class DiscoverAllCMS(object):
	# match all fingerprints against all responses
	# this might generate false positives

	def __init__(self, data):
		self.cache = data['cache']
		self.results = data['results']
		self.matcher = data['matcher']
		self.fps = data['fingerprints']
		self.printer = data['printer']

		# only used for pretty printing of debugging info
		self.tmp_set = set()

	def run(self):
		self.printer.print('Checking for more matches in cache (option -a)  ...', 1)

		# find matches for all the responses in the cache
		for fp_category in ['cms', 'platform']:
			for fp_type in self.fps.data[fp_category]:
				fps = self.fps.data[fp_category][fp_type]['fps']

				for response in self.cache.get_responses():
					matches = self.matcher.get_result(fps, response)
					for fp in matches:
						self.results.add( fp_category, fp['name'], fp['output'], fp )

						if (fp['name'], fp['output']) not in self.tmp_set:
							self.printer.print('- Found match: %s %s' % (fp['name'], fp['output']) , 2)

						self.tmp_set.add((fp['name'], fp['output']))


class DiscoverCMS(object):

	def __init__(self, options, data):
		self.printer = data['printer']
		self.matcher = data['matcher']
		self.requester = data['requester']
		self.result = data['results']
		self.printer = data['printer']

		self.batch_size = options['batch_size']
		self.num_cms_to_find = options['stop_after']
		self.find_all_cms = options['run_all']

		# only used for pretty printing of debugging info
		self.tmp_set = set()

		self.queue = defaultdict(list)
		for fp_type in data['fingerprints'].data['cms']:
			for fp in data['fingerprints'].data['cms'][fp_type]['fps']:
				self.queue[fp['url']].append(fp)



	def get_queue(self, cms=None):
		queue = []
		if cms is None:
			for i in range(self.batch_size):
				try:
					url, fp_list = self.queue.popitem()
					queue.append(fp_list)
				except KeyError:
					break
		else:
			# the following procedure is *not* optimal
			# the self.queue dict is completely destroyed and 
			# and rebuilt each time this procedure is called :(

			# create a temp queue dict
			tmp_queue = defaultdict(list)
			
			# remove elements from the dict until it is empty
			while len(self.queue) > 0:
				url, fp_list = self.queue.popitem()

				# remove all the elements of a queue entry's list
				# one-by-one and check if the fingerprints are
				# belong to the specified 'cms'
				tmp_list = []
				out_list = []
				
				while len(fp_list) > 0:
					# remove the fingerprint
					fp = fp_list.pop()

					# if the fingerprint matches the cms, add it to the 
					# out_list for the current url
					# otherwise add it to the tmp_list
					if fp['name'] == cms: 
						out_list.append(fp)
					else:
						tmp_list.append(fp)

				# if there are elements in tmp_list (the new list of fps that
				# that do *not* match the 'cms'), add it to the tmp_queue's entry
				# for the current url
				if len(tmp_list) > 0:
					tmp_queue[url].extend(tmp_list)

				# if matches for the specified cms have been found, add the list
				# to the fingerprintQueue for the requester
				if len(out_list) > 0:
					queue.append(out_list)

			# replace the queue with the tmp queue
			self.queue = tmp_queue

		return queue



	def run(self):
		batch_no = 0
		self.printer.print('Determining CMS type ...', 1)

		detected_cms = []
		stop_searching = len(detected_cms) >= self.num_cms_to_find

		while (not stop_searching or self.find_all_cms) and (not len(self.queue) == 0):
			self.printer.print('Checking fingerprint group no. %s ...' % (batch_no, ) , 3)

			# set the requester queue
			results = self.requester.run('CMS', self.get_queue())
			
			# search for CMS matches
			cms_matches = []
			while not results.empty():
				fingerprints, response = results.get()

				for fp in self.matcher.get_result(fingerprints, response):
					self.result.add( 'cms', fp['name'], fp['output'], fp)
					cms_matches.append(fp['name'])
		
			# search for the found CMS versions
			for cms in cms_matches:

				# skip checking the cms, if it has already been detected
				if cms in detected_cms: continue
{'fraction_non_alphanumeric': 0.11187890753537348, 'fraction_numerical': 0.0026324448831852583, 'mean_word_length': 3.449975597852611, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3176892', 'n_tokens_mistral': 3494, 'n_tokens_neox': 2862, 'n_words': 929}