<commit_before>require 'nokogiri'
require 'open-uri'
require 'open_uri_redirections'

class UsnReleaseNotes

  attr_reader :usn_id, :usn_title

  def initialize(usn_id)
    @usn_id    = usn_id.upcase
    @contents  = open(usn_url).read
    @doc       = Nokogiri::HTML(@contents)
    @usn_title = @doc.css('#main-content > div > h2').first.text
  end

  def text
    @text ||= release_note_text
  end

  private

  def usn_url
    "https://usn.ubuntu.com/#{@usn_id}/"
  end

  def release_note_text
    cves      = @doc.css('#main-content > div > h3:contains("References") + p > a[href*="cve/CVE"]')

    notes = "[#{usn_id}](#{usn_url}) #{usn_title}:\n"

    cves.each do |cve|
      cve_id  = cve.text
      cve_uri = cve['href']
      cve_element = Nokogiri::HTML(open(cve_uri, :allow_redirections => :safe).read).css('#container > div:contains("Description") > div.value').first

      cve_description = cve_element.children.map do |child|
        child.text if child.text?
      end.compact.join(" ")

      notes += "* [#{cve_id}](#{cve_uri}): #{cve_description}\n"
    end

    notes
  end
end<commit_msg>Update to match changes with Ubuntu USN publishing<commit_after>require 'nokogiri'
require 'open-uri'
require 'open_uri_redirections'

class UsnReleaseNotes

  attr_reader :usn_id, :usn_title

  def initialize(usn_id)
    @usn_id    = usn_id.upcase
    @contents  = open(usn_url).read
    @doc       = Nokogiri::HTML(@contents)
    @usn_title = @doc.css('#main-content h1').first.text
  end

  def text
    @text ||= release_note_text
  end

  private

  def usn_url
    "https://usn.ubuntu.com/#{@usn_id.gsub(/^USN-/, '')}/"
  end

  def release_note_text
    cves      = @doc.css('#references + ul > li > a[href*="cve/CVE"]')

    raise 'Could not find CVE references for release notes' if cves.empty?

    notes = "[#{usn_id}](#{usn_url}) #{usn_title}:\n"

    cves.each do |cve|
      cve_id  = cve.text
      cve_uri = cve['href']
      cve_element = Nokogiri::HTML(open(cve_uri, :allow_redirections => :safe).read).css('#container > div:contains("Description") > div.value').first

      cve_description = cve_element.children.map do |child|
        child.text if child.text?
      end.compact.join(" ")

      notes += "* [#{cve_id}](#{cve_uri}): #{cve_description}\n"
    end

    notes
  end
end
{'fraction_non_alphanumeric': 0.15421166306695464, 'fraction_numerical': 0.0012958963282937365, 'mean_word_length': 3.241758241758242, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2193309', 'n_tokens_mistral': 912, 'n_tokens_neox': 874, 'n_words': 161}