<commit_before>require 'risu'

ActiveRecord::Migration.verbose = false

module Risu
	module Parsers
		module Nessus
			
			# NessusSaxListener
			#
			# @author <NAME> <<EMAIL>>
			class NessusSaxListener
				include LibXML::XML::SaxParser::Callbacks

				# Sets up a array of all valid xml fields
				#
				def initialize
					@vals = Hash.new
					
					@valid_references = Array[
						"cpe", "bid", "see_also", "xref", "cve", "iava", "msft", 
						"osvdb", "cert", "edb-id", "rhsa", "secunia", "suse", "dsa", 
						"owasp", "cwe"]

					@valid_elements = Array["ReportItem", "plugin_version", "risk_factor",
						"description", "cvss_base_score", "solution", "item", "plugin_output", "tag", "synopsis", "plugin_modification_date",
						"FamilyName", "FamilyItem", "Status", "vuln_publication_date", "ReportHost", "HostProperties", "preferenceName",
						"preferenceValues", "preferenceType", "fullName", "pluginId", "pluginName", "selectedValue", "selectedValue",
						"name", "value", "preference", "plugin_publication_date", "cvss_vector", "patch_publication_date",
						"NessusClientData_v2", "Policy", "PluginName", "ServerPreferences", "policyComments", "policyName", "PluginItem",
						"Report", "Family", "Preferences", "PluginsPreferences", "FamilySelection", "IndividualPluginSelection", "PluginId",
						"pci-dss-compliance", "exploitability_ease", "cvss_temporal_vector", "exploit_framework_core", "cvss_temporal_score",
						"exploit_available", "metasploit_name", "exploit_framework_canvas", "canvas_package", "exploit_framework_metasploit",
						"plugin_type", "exploithub_sku", "exploit_framework_exploithub", "stig_severity", "plugin_name", "fname",
						]
						
						@valid_elements = @valid_elements + @valid_references
																		
						# This makes adding new host properties really easy, except for the 
						#MS patch numbers, this are handled differently.
						@valid_host_properties = {
							"HOST_END" => :end,
							"mac-address" => :mac,
							"HOST_START" => :start,
							"operating-system" => :os,
							"host-ip" => :ip,
							"host-fqdn" => :fqdn,
							"netbios-name" => :netbios,
							"local-checks-proto" => :local_checks_proto,
							"smb-login-used" => :smb_login_used,
							"ssh-auth-meth" => :ssh_auth_meth,
							"ssh-login-used" => :ssh_login_used,
							"pci-dss-compliance" => :pci_dss_compliance,
							"pci-dss-compliance:" => :pci_dss_compliance_ , #I think this is a Tenable bug~
							"pcidss:compliance:failed" => :pcidss_compliance_failed,
							"pcidss:compliance:passed" => :pcidss_compliance_passed,
							"pcidss:deprecated_ssl" => :pcidss_deprecated_ssl,
							"pcidss:expired_ssl_certificate" => :pcidss_expired_ssl_certificate,
							"pcidss:high_risk_flaw" => :pcidss_high_risk_flaw,
							"pcidss:medium_risk_flaw" => :pcidss_medium_risk_flaw,
							"pcidss:reachable_db" => :pcidss_reachable_db,
							"pcidss:www:xss" => :pcidss_www_xss,
							"pcidss:directory_browsing" => :pcidss_directory_browsing,
							"pcidss:known_credentials" => :pcidss_known_credentials,
							"pcidss:compromised_host:worm" => :pcidss_compromised_host_worm,
							"system-type" => :system_type,
							"pcidss:obsolete_operating_system" => :pcidss_obsolete_operating_system,
							"pcidss:dns_zone_transfer" => :pcidss_dns_zone_transfer,
							"pcidss:unprotected_mssql_db" => :pcidss_unprotected_mssql_db,
							"pcidss:obsolete_software" => :pcidss_obsolete_software,
							"pcidss:www:sql_injection" => :pcidss_www_sql_injection,
							"fname" => :fname
						}
				end

				# Callback for when the start of a xml element is reached
				#
				# @param element XML element
				# @param attributes Attributes for the XML element
				def on_start_element(element, attributes)
					@tag = element
					@vals[@tag] = ""

					if !@valid_elements.include?(element)
						puts "New XML element detected: #{element}. Please report this to #{Risu::EMAIL}"
					end

					case element
						when "Policy"
							@policy = Risu::Models::Policy.create
							@policy.save
						when "preference"
							@sp = @policy.server_preferences.create
							@sp.save
						when "item"
							@item = @policy.plugins_preferences.create
							@item.save
						when "FamilyItem"
							@family = @policy.family_selections.create
							@family.save
						when "PluginItem"
							@plugin_selection = @policy.individual_plugin_selections.create
							@plugin_selection.save
						when "Report"
							@report = @policy.reports.create
							@report.name = attributes["name"]
							@report.save
						when "ReportHost"
							@rh = @report.hosts.create
							@rh.name = attributes["name"]
							@rh.save
						when "tag"
							@attr = nil

							if attributes["name"] =~ /[M|m][S|s]\d{2}-\d{2,}/
								@attr = if attributes["name"] =~ /[M|m][S|s]\d{2}-\d{2,}/
										attributes["name"]
									else
										nil
									end
							else
								@attr = if @valid_host_properties.keys.include?(attributes["name"])
										attributes["name"]
									else
										nil
									end
							end

							puts "New HostProperties attribute: #{attributes["name"]}. Please report this to <EMAIL>\n" if @attr.nil?
						when "ReportItem"
							@vals = Hash.new # have to clear this out or everything has the same references
							@ri = @rh.items.create
							if attributes["pluginID"] == "0"
								@plugin = Risu::Models::Plugin.find_or_create_by_id(1)
							else
								@plugin = Risu::Models::Plugin.find_or_create_by_id(attributes["pluginID"])
							end
<commit_msg>Fixed a issue with MS Host Property tags<commit_after>require 'risu'

ActiveRecord::Migration.verbose = false

module Risu
	module Parsers
		module Nessus
			
			# NessusSaxListener
			#
			# @author <NAME> <<EMAIL>>
			class NessusSaxListener
				include LibXML::XML::SaxParser::Callbacks

				# Sets up a array of all valid xml fields
				#
				def initialize
					@vals = Hash.new
					
					@valid_references = Array[
						"cpe", "bid", "see_also", "xref", "cve", "iava", "msft", 
						"osvdb", "cert", "edb-id", "rhsa", "secunia", "suse", "dsa", 
						"owasp", "cwe"]

					@valid_elements = Array["ReportItem", "plugin_version", "risk_factor",
						"description", "cvss_base_score", "solution", "item", "plugin_output", "tag", "synopsis", "plugin_modification_date",
						"FamilyName", "FamilyItem", "Status", "vuln_publication_date", "ReportHost", "HostProperties", "preferenceName",
						"preferenceValues", "preferenceType", "fullName", "pluginId", "pluginName", "selectedValue", "selectedValue",
						"name", "value", "preference", "plugin_publication_date", "cvss_vector", "patch_publication_date",
						"NessusClientData_v2", "Policy", "PluginName", "ServerPreferences", "policyComments", "policyName", "PluginItem",
						"Report", "Family", "Preferences", "PluginsPreferences", "FamilySelection", "IndividualPluginSelection", "PluginId",
						"pci-dss-compliance", "exploitability_ease", "cvss_temporal_vector", "exploit_framework_core", "cvss_temporal_score",
						"exploit_available", "metasploit_name", "exploit_framework_canvas", "canvas_package", "exploit_framework_metasploit",
						"plugin_type", "exploithub_sku", "exploit_framework_exploithub", "stig_severity", "plugin_name", "fname",
						]
						
						@valid_elements = @valid_elements + @valid_references
																		
						# This makes adding new host properties really easy, except for the 
						#MS patch numbers, this are handled differently.
						@valid_host_properties = {
							"HOST_END" => :end,
							"mac-address" => :mac,
							"HOST_START" => :start,
							"operating-system" => :os,
							"host-ip" => :ip,
							"host-fqdn" => :fqdn,
							"netbios-name" => :netbios,
							"local-checks-proto" => :local_checks_proto,
							"smb-login-used" => :smb_login_used,
							"ssh-auth-meth" => :ssh_auth_meth,
							"ssh-login-used" => :ssh_login_used,
							"pci-dss-compliance" => :pci_dss_compliance,
							"pci-dss-compliance:" => :pci_dss_compliance_ , #I think this is a Tenable bug~
							"pcidss:compliance:failed" => :pcidss_compliance_failed,
							"pcidss:compliance:passed" => :pcidss_compliance_passed,
							"pcidss:deprecated_ssl" => :pcidss_deprecated_ssl,
							"pcidss:expired_ssl_certificate" => :pcidss_expired_ssl_certificate,
							"pcidss:high_risk_flaw" => :pcidss_high_risk_flaw,
							"pcidss:medium_risk_flaw" => :pcidss_medium_risk_flaw,
							"pcidss:reachable_db" => :pcidss_reachable_db,
							"pcidss:www:xss" => :pcidss_www_xss,
							"pcidss:directory_browsing" => :pcidss_directory_browsing,
							"pcidss:known_credentials" => :pcidss_known_credentials,
							"pcidss:compromised_host:worm" => :pcidss_compromised_host_worm,
							"system-type" => :system_type,
							"pcidss:obsolete_operating_system" => :pcidss_obsolete_operating_system,
							"pcidss:dns_zone_transfer" => :pcidss_dns_zone_transfer,
							"pcidss:unprotected_mssql_db" => :pcidss_unprotected_mssql_db,
							"pcidss:obsolete_software" => :pcidss_obsolete_software,
							"pcidss:www:sql_injection" => :pcidss_www_sql_injection,
							"fname" => :fname
						}
				end

				# Callback for when the start of a xml element is reached
				#
				# @param element XML element
				# @param attributes Attributes for the XML element
				def on_start_element(element, attributes)
					@tag = element
					@vals[@tag] = ""

					if !@valid_elements.include?(element)
						puts "New XML element detected: #{element}. Please report this to #{Risu::EMAIL}"
					end

					case element
						when "Policy"
							@policy = Risu::Models::Policy.create
							@policy.save
						when "preference"
							@sp = @policy.server_preferences.create
							@sp.save
						when "item"
							@item = @policy.plugins_preferences.create
							@item.save
						when "FamilyItem"
							@family = @policy.family_selections.create
							@family.save
						when "PluginItem"
							@plugin_selection = @policy.individual_plugin_selections.create
							@plugin_selection.save
						when "Report"
							@report = @policy.reports.create
							@report.name = attributes["name"]
							@report.save
						when "ReportHost"
							@rh = @report.hosts.create
							@rh.name = attributes["name"]
							@rh.save
						when "tag"
							@attr = nil

							if attributes["name"] =~ /[M|m][S|s]\d{2,}-\d{2,}/
								@attr = if attributes["name"] =~ /[M|m][S|s]\d{2,}-\d{2,}/
										attributes["name"]
									else
										nil
									end
							else
								@attr = if @valid_host_properties.keys.include?(attributes["name"])
										attributes["name"]
									else
										nil
									end
							end

							puts "New HostProperties attribute: #{attributes["name"]}. Please report this to <EMAIL>\n" if @attr.nil?
						when "ReportItem"
							@vals = Hash.new # have to clear this out or everything has the same references
							@ri = @rh.items.create
							if attributes["pluginID"] == "0"
								@plugin = Risu::Models::Plugin.find_or_create_by_id(1)
							else
								@plugin = Risu::Models::Plugin.find_or_create_by_id(attributes["pluginID"])
							end

{'fraction_non_alphanumeric': 0.13554459883460332, 'fraction_numerical': 0.0012550425818018825, 'mean_word_length': 3.2482863670982485, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5316925', 'n_tokens_mistral': 5170, 'n_tokens_neox': 3689, 'n_words': 703}