<commit_before>module CveEngine

  require 'cve_engine/schema/vulnerability/vulnerability_type'

  # NVD Vulnerability
  #
  # rails g model Vulnerability cve_id:string is_cce_id:boolean discovered_datetime:datetime disclosure_datetime:datetime exploit_publish_datetime:datetime published_datetime:datetime last_modified_datetime:datetime security_protection:string summary:string
  #
  # app/assets/schema/vulnerability_0.4.xsd
  #
  # ```xml
  #<xsd:element name="osvdb-ext" type="osvdbExtensionType" minOccurs="0"/>
  #<xsd:element name="vulnerable-configuration" type="cpe-lang:PlatformType" minOccurs="0" maxOccurs="unbounded"/>
  #<xsd:element name="vulnerable-software-list" type="vulnerableSoftwareType" minOccurs="0"/>
  #<xsd:choice minOccurs="0">
  #  <xsd:element name="cve-id" type="cve:cveNamePatternType"/>
  #  <xsd:element name="cce-id" type="cce:cceNamePatternType"/>
  #</xsd:choice>
  #<xsd:element name="discovered-datetime" type="xsd:dateTime" minOccurs="0"/>
  #<xsd:element name="disclosure-datetime" type="xsd:dateTime" minOccurs="0"/>
  #<xsd:element name="exploit-publish-datetime" type="xsd:dateTime" minOccurs="0"/>
  #<xsd:element name="published-datetime" type="xsd:dateTime" minOccurs="0"/>
  #<xsd:element name="last-modified-datetime" type="xsd:dateTime" minOccurs="0"/>
  #<xsd:element name="cvss" type="cvssv2:cvssImpactType" minOccurs="0"/>
  #<xsd:element name="security-protection" type="securityProtectionType" minOccurs="0"/>
  #<xsd:element name="assessment_check" type="scap-core:checkReferenceType" maxOccurs="unbounded" minOccurs="0"/>
  #<xsd:element name="cwe" type="cweReferenceType" minOccurs="0" maxOccurs="unbounded"/>
  #<xsd:element name="references" type="vulnerabilityReferenceType" minOccurs="0" maxOccurs="unbounded"/>
  #<xsd:element name="fix_action" type="fixActionType" minOccurs="0" maxOccurs="unbounded"/>
  #<xsd:element name="scanner" type="toolConfigurationType" minOccurs="0" maxOccurs="unbounded" />
  #<xsd:element name="summary" type="xsd:string" minOccurs="0"/>
  #<xsd:element name="technical_description" type="scap-core:referenceType" minOccurs="0" maxOccurs="unbounded"/>
  #<xsd:element name="attack_scenario" type="scap-core:referenceType" minOccurs="0" maxOccurs="unbounded" />
  #```
  class Vulnerability < ActiveRecord::Base
    # -----------------------------------------------------
    # Constants
    SECURITY_PROTECTIONS = ["ALLOWS_ADMIN_ACCESS", "ALLOWS_USER_ACCESS", "ALLOWS_OTHER_ACCESS"]
    CVE_ID_REGEX = /CVE-\d{4}-\d{4}/
    # -----------------------------------------------------
    # Inclusion / Extension (acts_as)

    # -----------------------------------------------------
    # Associations
    #TODO has_one :osvdb_ext
    #TODO has_one :cvss

    #TODO has_many :vulnerable_configurations
    has_and_belongs_to_many :cpe_products
    #TODO has_many :assessment_checks
    #TODO has_many :cwes
    #TODO has_many :references
    #TODO has_many :fix_actions
    #TODO has_many :scanners
    #TODO has_many :technical_descriptions
    #TODO has_many :attack_scenarios

    # -----------------------------------------------------
    # Validations
    validates :cve_id, presence: true, uniqueness: true, format: {with: CVE_ID_REGEX}
    validates :security_protection, inclusion: {in: SECURITY_PROTECTIONS, allow_nil: true}

    # -----------------------------------------------------
    # Hooks

    # -----------------------------------------------------
    # Instance methods
    def security_protection_enum
      SECURITY_PROTECTIONS
    end

    #def vulnerable_software_list
    #  self.cpe_products
    #end

    # -----------------------------------------------------
    # Class methods
    def self.find_from_xml(data)
      xml_vuln = Schema::Vulnerability::VulnerabilityType.from_xml(data)
      vuln = self.find_by cve_id: vuln.id
    end

    def self.create_from_xml(data)
      xml_vuln = Schema::Vulnerability::VulnerabilityType.from_xml(data)
      vuln = Vulnerability.create(
          cve_id: xml_vuln.id,
          is_cce_id: xml_vuln.is_cce_id?,
          discovered_datetime: xml_vuln.discovered_datetime,
          disclosure_datetime: xml_vuln.disclosure_datetime,
          exploit_publish_datetime: xml_vuln.exploit_publish_datetime,
          published_datetime: xml_vuln.published_datetime,
          last_modified_datetime: xml_vuln.last_modified_datetime,
          security_protection: xml_vuln.security_protection,
          summary: xml_vuln.summary
      )
      if vuln
        xml_vuln.vulnerable_software_list.products.each do |xml_cpe_product|
          vuln.cpe_products << CpeProduct.create(cpe_name: xml_cpe_product)
        end
        vuln.save
      end
      vuln
    end

    def self.update_from_xml(data)
      xml_vuln = Schema::Vulnerability::VulnerabilityType.from_xml(data)
      vuln = self.find_by cve_id: vuln.id
      if vuln
        vuln.update!(
            is_cce_id: xml_vuln.is_cce_id?,
            discovered_datetime: xml_vuln.discovered_datetime,
            disclosure_datetime: xml_vuln.disclosure_datetime,
            exploit_publish_datetime: xml_vuln.exploit_publish_datetime,
            published_datetime: xml_vuln.published_datetime,
            last_modified_datetime: xml_vuln.last_modified_datetime,
            security_protection: xml_vuln.security_protection,
            summary: xml_vuln.summary
        )

        vuln.cpe_products = []
        xml_vuln.vulnerable_software_list.products.each do |xml_cpe_product|
          vuln.cpe_products << CpeProduct.create_from_cpe_name(xml_cpe_product)
        end
        vuln.save
      end

    end

    def self.find_or_create_from_xml(data)
      xml_vuln = Schema::Vulnerability::VulnerabilityType.from_xml(data)
      vuln = self.find_by cve_id: xml_vuln.id
      vuln = self.create_from_xml(data) unless vuln
      vuln
    end


    # -----------------------------------------------------
    # Private methods
    private

    # -----------------------------------------------------
    # rails_admin configuration
    rails_admin do
      list do
        field :cve_id
        field :is_cce_id
        field :discovered_datetime
        field :disclosure_datetime
        field :exploit_publish_datetime
        field :published_datetime
        field :last_modified_datetime
        field :security_protection
        field :summary
      end

      show do
        field :cve_id
        field :is_cce_id
        field :discovered_datetime
        field :disclosure_datetime
        field :exploit_publish_datetime
        field :published_datetime
        field :last_modified_datetime
        field :security_protection
        field :summary
      end

      edit do
        field :cve_id
        field :is_cce_id
        field :discovered_datetime
        field :disclosure_datetime
        field :exploit_publish_datetime
        field :published_datetime
        field :last_modified_datetime
        field :security_protection
        field :summary
      end

      create do
        field :cve_id
        field :is_cce_id
        field :discovered_datetime
        field :disclosure_datetime
        field :exploit_publish_datetime
        field :published_datetime
        field :last_modified_datetime
        field :security_protection
        field :summary
      end

      update do
        field :cve_id
        field :is_cce_id
        field :discovered_datetime
        field :disclosure_datetime
        field :exploit_publish_datetime
        field :published_datetime
        field :last_modified_datetime
        field :security_protection
        field :summary
      end

      nested do
        field :cve_id
        field :is_cce_id
        field :discovered_datetime
        field :disclosure_datetime
        field :exploit_publish_datetime
        field :published_datetime
        field :last_modified_datetime
        field :security_protection
        field :summary
      end

      modal do
        field :cve_id
        field :is_cce_id
        field :discovered_datetime
        field :disclosure_datetime
        field :exploit_publish_datetime
        field :published_datetime
        field :last_modified_datetime
        field :security_protection
        field :summary
      end
    end
  end
end
<commit_msg>model: fixes CPE product duplication issue<commit_after>module CveEngine

  require 'cve_engine/schema/vulnerability/vulnerability_type'

  # NVD Vulnerability
  #
  # rails g model Vulnerability cve_id:string is_cce_id:boolean discovered_datetime:datetime disclosure_datetime:datetime exploit_publish_datetime:datetime published_datetime:datetime last_modified_datetime:datetime security_protection:string summary:string
  #
  # app/assets/schema/vulnerability_0.4.xsd
  #
  # ```xml
  #<xsd:element name="osvdb-ext" type="osvdbExtensionType" minOccurs="0"/>
  #<xsd:element name="vulnerable-configuration" type="cpe-lang:PlatformType" minOccurs="0" maxOccurs="unbounded"/>
  #<xsd:element name="vulnerable-software-list" type="vulnerableSoftwareType" minOccurs="0"/>
  #<xsd:choice minOccurs="0">
  #  <xsd:element name="cve-id" type="cve:cveNamePatternType"/>
  #  <xsd:element name="cce-id" type="cce:cceNamePatternType"/>
  #</xsd:choice>
  #<xsd:element name="discovered-datetime" type="xsd:dateTime" minOccurs="0"/>
  #<xsd:element name="disclosure-datetime" type="xsd:dateTime" minOccurs="0"/>
  #<xsd:element name="exploit-publish-datetime" type="xsd:dateTime" minOccurs="0"/>
  #<xsd:element name="published-datetime" type="xsd:dateTime" minOccurs="0"/>
  #<xsd:element name="last-modified-datetime" type="xsd:dateTime" minOccurs="0"/>
  #<xsd:element name="cvss" type="cvssv2:cvssImpactType" minOccurs="0"/>
  #<xsd:element name="security-protection" type="securityProtectionType" minOccurs="0"/>
  #<xsd:element name="assessment_check" type="scap-core:checkReferenceType" maxOccurs="unbounded" minOccurs="0"/>
  #<xsd:element name="cwe" type="cweReferenceType" minOccurs="0" maxOccurs="unbounded"/>
  #<xsd:element name="references" type="vulnerabilityReferenceType" minOccurs="0" maxOccurs="unbounded"/>
  #<xsd:element name="fix_action" type="fixActionType" minOccurs="0" maxOccurs="unbounded"/>
  #<xsd:element name="scanner" type="toolConfigurationType" minOccurs="0" maxOccurs="unbounded" />
  #<xsd:element name="summary" type="xsd:string" minOccurs="0"/>
  #<xsd:element name="technical_description" type="scap-core:referenceType" minOccurs="0" maxOccurs="unbounded"/>
  #<xsd:element name="attack_scenario" type="scap-core:referenceType" minOccurs="0" maxOccurs="unbounded" />
  #```
  class Vulnerability < ActiveRecord::Base
    # -----------------------------------------------------
    # Constants
    SECURITY_PROTECTIONS = ["ALLOWS_ADMIN_ACCESS", "ALLOWS_USER_ACCESS", "ALLOWS_OTHER_ACCESS"]
    CVE_ID_REGEX = /CVE-\d{4}-\d{4}/
    # -----------------------------------------------------
    # Inclusion / Extension (acts_as)

    # -----------------------------------------------------
    # Associations
    #TODO has_one :osvdb_ext
    #TODO has_one :cvss

    #TODO has_many :vulnerable_configurations
    has_and_belongs_to_many :cpe_products
    #TODO has_many :assessment_checks
    #TODO has_many :cwes
    #TODO has_many :references
    #TODO has_many :fix_actions
    #TODO has_many :scanners
    #TODO has_many :technical_descriptions
    #TODO has_many :attack_scenarios

    # -----------------------------------------------------
    # Validations
    validates :cve_id, presence: true, uniqueness: true, format: {with: CVE_ID_REGEX}
    validates :security_protection, inclusion: {in: SECURITY_PROTECTIONS, allow_nil: true}

    # -----------------------------------------------------
    # Hooks

    # -----------------------------------------------------
    # Instance methods
    def security_protection_enum
      SECURITY_PROTECTIONS
    end

    #def vulnerable_software_list
    #  self.cpe_products
    #end

    # -----------------------------------------------------
    # Class methods
    def self.find_from_xml(data)
      xml_vuln = Schema::Vulnerability::VulnerabilityType.from_xml(data)
      vuln = self.find_by cve_id: vuln.id
    end

    def self.create_from_xml(data)
      xml_vuln = Schema::Vulnerability::VulnerabilityType.from_xml(data)
      vuln = Vulnerability.create(
          cve_id: xml_vuln.id,
          is_cce_id: xml_vuln.is_cce_id?,
          discovered_datetime: xml_vuln.discovered_datetime,
          disclosure_datetime: xml_vuln.disclosure_datetime,
          exploit_publish_datetime: xml_vuln.exploit_publish_datetime,
          published_datetime: xml_vuln.published_datetime,
          last_modified_datetime: xml_vuln.last_modified_datetime,
          security_protection: xml_vuln.security_protection,
          summary: xml_vuln.summary
      )
      if vuln
        xml_vuln.vulnerable_software_list.products.each do |xml_cpe_product|
          vuln.cpe_products << CpeProduct.find_or_create_by(cpe_name: xml_cpe_product)
        end if xml_vuln.vulnerable_software_list
        vuln.save
      end
      vuln
    end

    def self.update_from_xml(data)
      xml_vuln = Schema::Vulnerability::VulnerabilityType.from_xml(data)
      vuln = self.find_by cve_id: vuln.id
      if vuln
        vuln.update!(
            is_cce_id: xml_vuln.is_cce_id?,
            discovered_datetime: xml_vuln.discovered_datetime,
            disclosure_datetime: xml_vuln.disclosure_datetime,
            exploit_publish_datetime: xml_vuln.exploit_publish_datetime,
            published_datetime: xml_vuln.published_datetime,
            last_modified_datetime: xml_vuln.last_modified_datetime,
            security_protection: xml_vuln.security_protection,
            summary: xml_vuln.summary
        )

        vuln.cpe_products = []
        xml_vuln.vulnerable_software_list.products.each do |xml_cpe_product|
          vuln.cpe_products << CpeProduct.create_from_cpe_name(xml_cpe_product)
        end
        vuln.save
      end

    end

    def self.find_or_create_from_xml(data)
      xml_vuln = Schema::Vulnerability::VulnerabilityType.from_xml(data)
      vuln = self.find_by cve_id: xml_vuln.id
      vuln = self.create_from_xml(data) unless vuln
      vuln
    end


    # -----------------------------------------------------
    # Private methods
    private

    # -----------------------------------------------------
    # rails_admin configuration
    rails_admin do
      list do
        field :cve_id
        field :is_cce_id
        field :discovered_datetime
        field :disclosure_datetime
        field :exploit_publish_datetime
        field :published_datetime
        field :last_modified_datetime
        field :security_protection
        field :summary
      end

      show do
        field :cve_id
        field :is_cce_id
        field :discovered_datetime
        field :disclosure_datetime
        field :exploit_publish_datetime
        field :published_datetime
        field :last_modified_datetime
        field :security_protection
        field :summary
      end

      edit do
        field :cve_id
        field :is_cce_id
        field :discovered_datetime
        field :disclosure_datetime
        field :exploit_publish_datetime
        field :published_datetime
        field :last_modified_datetime
        field :security_protection
        field :summary
      end

      create do
        field :cve_id
        field :is_cce_id
        field :discovered_datetime
        field :disclosure_datetime
        field :exploit_publish_datetime
        field :published_datetime
        field :last_modified_datetime
        field :security_protection
        field :summary
      end

      update do
        field :cve_id
        field :is_cce_id
        field :discovered_datetime
        field :disclosure_datetime
        field :exploit_publish_datetime
        field :published_datetime
        field :last_modified_datetime
        field :security_protection
        field :summary
      end

      nested do
        field :cve_id
        field :is_cce_id
        field :discovered_datetime
        field :disclosure_datetime
        field :exploit_publish_datetime
        field :published_datetime
        field :last_modified_datetime
        field :security_protection
        field :summary
      end

      modal do
        field :cve_id
        field :is_cce_id
        field :discovered_datetime
        field :disclosure_datetime
        field :exploit_publish_datetime
        field :published_datetime
        field :last_modified_datetime
        field :security_protection
        field :summary
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.1385972280554389, 'fraction_numerical': 0.002879942401151977, 'mean_word_length': 2.5638229634381013, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 77}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2350103', 'n_tokens_mistral': 5246, 'n_tokens_neox': 5050, 'n_words': 887}