<commit_before>require "pathname"

Ohai.plugin(:Hardware) do
  provides "hardware"

  def read_sysctl_link(file)
    File.basename(File.readlink(file))
  rescue Errno::ENOENT, Errno::ENOTDIR
  end

  def read_sysctl_file(file)
    IO.read(file).strip
  rescue Errno::ENOENT, Errno::ENOTDIR, Errno::EINVAL
  end

  def parse_memory_size(size)
    if size =~ /^(\d+(?:\.\d+)?)\s*TB/i
      Regexp.last_match(1).to_f * 2**30
    elsif size =~ /^(\d+(?:\.\d+)?)\s*GB/i
      Regexp.last_match(1).to_f * 2**20
    elsif size =~ /^(\d+(?:\.\d+)?)\s*MB/i
      Regexp.last_match(1).to_f * 2**10
    end
  end

  def format_disk_size(kb)
    if kb == 0
      ""
    else
      kblog10 = Math.log10(kb).floor

      kb = kb.to_f * 2 / 10**kblog10
      kb = kb.round.to_f / 2

      if kblog10 >= 9
        format "%gTB", kb * 10**(kblog10 - 9)
      elsif kblog10 >= 6
        format "%dGB", kb * 10**(kblog10 - 6)
      else
        format "%dMB", kb * 10**(kblog10 - 3)
      end
    end
  end

  def memory_to_disk_size(size)
    format_disk_size(parse_memory_size(size))
  end

  def find_sas_device(address)
    file = Dir.glob("/sys/class/scsi_generic/sg*/device/sas_address").find do |file|
      read_sysctl_file(file) == "0x#{address}"
    end

    if file
      dir = File.dirname(file)
      device = Dir.glob("#{dir}/block/*").first ||
               Dir.glob("#{dir}/scsi_generic/*").first

      "/dev/#{File.basename(device)}"
    end
  end

  def pci_devices
    devices = {}
    device = nil

    IO.popen(["lspci", "-Dkvmm"]).each do |line|
      if line =~ /^Slot:\s+((\h{4}):(\h{2}):(\h{2}).(\h))\s*$/
        device = {
          :slot => Regexp.last_match(1),
          :domain => Regexp.last_match(2),
          :bus => Regexp.last_match(3),
          :device => Regexp.last_match(4),
          :function => Regexp.last_match(5)
        }
      elsif device && line =~ /^([A-Z]+):\s+(.*)\s*$/i
        case Regexp.last_match(1)
        when "Class" then device[:class_name] = Regexp.last_match(2)
        when "Vendor" then device[:vendor_name] = Regexp.last_match(2)
        when "Device" then device[:device_name] = Regexp.last_match(2)
        when "SVendor" then device[:subsystem_vendor_name] = Regexp.last_match(2)
        when "SDevice" then device[:subsystem_device_name] = Regexp.last_match(2)
        when "PhySlot" then device[:physical_slot] = Regexp.last_match(2)
        when "Rev" then device[:revision] = Regexp.last_match(2)
        when "ProgIf" then device[:programming_interface] = Regexp.last_match(2)
        when "Driver" then device[:driver] = Regexp.last_match(2)
        when "Module" then device[:modules] = Array(device[:modules]) << Regexp.last_match(2)
        end
      elsif device && line =~ /^\s*$/
        devices[device[:slot]] = device
        device = nil
      end
    end

    IO.popen(["lspci", "-Dkvmmn"]).each do |line|
      if line =~ /^Slot:\s+((\h{4}):(\h{2}):(\h{2}).(\h))\s*$/
        device = devices[Regexp.last_match(1)]
      elsif device && line =~ /^([A-Z]+):\s+(.*)\s*$/i
        case Regexp.last_match(1)
        when "Class" then device[:class_id] = Regexp.last_match(2)
        when "Vendor" then device[:vendor_id] = Regexp.last_match(2)
        when "Device" then device[:device_id] = Regexp.last_match(2)
        when "SVendor" then device[:subsystem_vendor_id] = Regexp.last_match(2)
        when "SDevice" then device[:subsystem_device_id] = Regexp.last_match(2)
        end
      elsif device && line =~ /^\s*$/
        device = nil
      end
    end

    devices
  end

  def network_devices
    Dir.glob("/sys/class/net/*").each_with_object(Mash.new) do |device, devices|
      name = File.basename(device)

      devices[name] = {
        :device => read_sysctl_link("#{device}/device"),
        :duplex => read_sysctl_file("#{device}/duplex"),
        :speed => read_sysctl_file("#{device}/speed")
      }.delete_if { |_, v| v.nil? }
    end
  end

  def memory_devices
    device = nil

    IO.popen(["dmidecode", "-t", "memory"]).each_with_object([]) do |line, devices|
      if line =~ /^Memory Device\s*$/
        device = {}
      elsif device && line =~ /^\s+([A-Z ]+):\s+(.*)\s*$/i
        device[Regexp.last_match(1).tr(" ", "_").downcase.to_sym] = Regexp.last_match(2).strip
      elsif device && line =~ /^\s*$/
        devices << device
        device = nil
      end
    end
  end

  def disk_devices
    disk = Mash.new

    disk[:controllers] = []
    disk[:arrays] = []
    disk[:disks] = []

    find_direct_disks(disk)
    find_nvme_disks(disk)

    find_hp_disks(disk) if File.exist?("/usr/sbin/ssacli")
    find_megaraid_disks(disk) if File.exist?("/usr/sbin/megacli")
    find_mpt1_disks(disk) if File.exist?("/usr/sbin/lsiutil")
    find_mpt2_disks(disk) if File.exist?("/usr/sbin/sas2ircu")
    find_adaptec_disks(disk) if File.exist?("/usr/sbin/arcconf")
    find_areca_disks(disk) if File.exist?("/opt/areca/x86_64/cli64")

    find_md_arrays(disk)

    disk[:disks].each do |disk|
      if disk[:vendor] =~ /^(BTWA|CVPR|PHDV)/ && disk[:model] == "INTEL"
        disk[:model] = disk[:serial_number]
        disk[:serial_number] = disk[:vendor]
        disk[:vendor] = "INTEL"
      end

      if disk[:vendor].nil? && disk[:model] =~ /^ATA\s+(.*)$/
        disk[:vendor] = "ATA"
        disk[:model] = Regexp.last_match(1)
      end

      if disk[:vendor].nil? || disk[:vendor] == "ATA"
        if disk[:model] =~ /^(\S+)\s+(.*)$/
          disk[:vendor] = Regexp.last_match(1)
          disk[:model] = Regexp.last_match(2)
        elsif disk[:model] =~ /^ST/
          disk[:vendor] = "SEAGATE"
        elsif disk[:model] =~ /^C300-(.*)$/
          disk[:vendor] = "CRUCIAL"
          disk[:model] = Regexp.last_match(1)
        end
      end

      disk[:model].sub!(/-.*$/, "") if disk[:model]
    end

    disk
  end

  def find_direct_disks(devices)
    Dir.glob("/sys/class/scsi_host/host*") do |host|
      driver = read_sysctl_file("#{host}/proc_name")

      if %w(ahci mptsas sata_mv sata_nv).include?(driver)
        bus = host.sub("/sys/class/scsi_host/host", "")

        Dir.glob("/sys/bus/scsi/devices/#{bus}:0:*").each do |device|
          next unless File.exist?("#{device}/scsi_disk")

          block = Dir.glob("#{device}/block/*").first
          size = read_sysctl_file("#{block}/size").to_f / 2

          devices[:disks] << {
            :id => devices[:disks].count,
            :device => "/dev/#{File.basename(block)}",
            :vendor => read_sysctl_file("#{device}/vendor"),
            :model => read_sysctl_file("#{device}/model"),
            :firmware_version => read_sysctl_file("#{device}/rev"),
            :size => format_disk_size(size),
            :arrays => []
          }
        end
      end
    end
  end

  def find_nvme_disks(devices)
    Dir.glob("/sys/class/nvme/nvme*") do |device|
      controller = {
        :id => devices[:controllers].count,
        :pci_slot => File.basename(Pathname.new("#{device}/device").realpath),
        :arrays => [],
        :disks => []
      }

      devices[:controllers] << controller

      IO.popen(["lspci", "-Dkvmm", "-s", controller[:pci_slot]]).each do |line|
        if line =~ /^SVendor:\s+(\S.*\S)\s*$/
          controller[:vendor] = Regexp.last_match(1)
        elsif line =~ /^SDevice:\s+(\S.*\S)\s*$/
          controller[:model] = Regexp.last_match(1)
        end
      end

      Dir.glob("#{device}/nvme*").each do |block|
        size = read_sysctl_file("#{block}/size").to_f / 2

        disk = {
          :id => devices[:disks].count,
          :controller => controller[:id],
          :device => "/dev/#{File.basename(block)}",
          :vendor => controller[:vendor],
          :model => controller[:model],
          :size => format_disk_size(size),
          :arrays => []
        }

        devices[:disks] << disk
        controller[:disks] << disk[:id]
      end
    end
  end

  def find_md_arrays(devices)
    array = nil

    File.new("/proc/mdstat", "r").each do |line|
      if line =~ /^(md\d+) : active raid(\d+)((?: (?:sd[a-z]\d*|nvme\d+n\d+(?:p\d+)?)\[\d+\](?:\([A-Z]\))*)+)$/
        array = {
          :id => devices[:arrays].count,
          :device => "/dev/#{Regexp.last_match(1)}",
          :status => "optimal",
          :raid_level => Regexp.last_match(2),
          :disks => []
        }

        Regexp.last_match(3).split(" ").each do |member|
          if member =~ /^(sd[a-z]+|nvme\d+n\d+).*/
            device = Regexp.last_match(1)

            if disk = devices[:disks].find { |d| d[:device] == "/dev/#{device}" }
              if member =~ /\(F\)/
                disk[:status] = "failed"
              elsif member =~ /\(S\)/
                disk[:status] = "hotspare"
              else
                disk[:status] = "online"
              end

              disk[:arrays] << array[:id]
              array[:disks] << disk[:id]
            end
          end
        end

        devices[:arrays] << array
      elsif array && line =~ /^\s+(\d+) blocks.*(?:\[([U_]+)\])?/
        array[:size] = format_disk_size(Regexp.last_match(1).to_i)
        array[:status] = "degraded" if Regexp.last_match(2) =~ /_/
      elsif array && line =~ /^\s+\[.*\]\s+(\S+)\s+=/
        case Regexp.last_match(1)
        when "recovery" then array[:status] = "rebuilding"
        when "resync" then array[:status] = "rebuilding"
        when "checking" then array[:status] = "checking"
        end
      end
    end
  end

  def find_hp_disks(devices)
    controllers = []
    disks = []

    controller = nil
    array = nil
    disk = nil

    IO.popen(%w(ssacli controller all show config detail)).each do |line|
      next unless line.valid_encoding?

      if line =~ /^Smart (?:Array|HBA) (\S+) /
        controller = {
          :id => devices[:controllers].count,
          :type => "hp",
          :model => Regexp.last_match(1),
          :arrays => [],
          :disks => []
        }

        devices[:controllers] << controller

        controllers << controller

        array = nil
        disk = nil
      elsif controller && line =~ /^   (\S.*):\s+(.*)$/
        case Regexp.last_match(1)
        when "Slot" then controller[:slot] = Regexp.last_match(2)
        when "Serial Number" then controller[:serial_number] = Regexp.last_match(2)
        when "Hardware Revision" then controller[:hardware_version] = Regexp.last_match(2)
        when "Firmware Version" then controller[:firmware_version] = Regexp.last_match(2)
        when "PCI Address (Domain:Bus:Device.Function)" then controller[:pci_slot] = Regexp.last_match(2)
        end
      elsif controller && line =~ /^      Logical Drive: (\d+)$/
        array = {
          :id => devices[:arrays].count,
          :controller => controller[:id],
          :number => Regexp.last_match(1),
          :disks => []
        }

        devices[:arrays] << array
        controller[:arrays] << array[:id]

        disk = nil
      elsif array && line =~ /^      physicaldrive (\S+)$/
        disk = {
          :id => devices[:disks].count,
          :controller => controller[:id],
          :arrays => [array[:id]],
          :location => Regexp.last_match(1)
        }

        devices[:disks] << disk
        controller[:disks] << disk[:id]
        array[:disks] << disk[:id]
      elsif disk && line =~ /^         (\S[^:]+):\s+(.*\S)\s*$/
        case Regexp.last_match(1)
        when "Status" then disk[:status] = Regexp.last_match(2)
        when "Drive Type" then disk[:drive_type] = Regexp.last_match(2)
        when "Interface Type" then disk[:interface] = Regexp.last_match(2)
        when "Size" then disk[:size] = Regexp.last_match(2)
        when "Rotational Speed" then disk[:rpm] = Regexp.last_match(2)
        when "Firmware Revision" then disk[:firmware_version] = Regexp.last_match(2)
        when "Serial Number" then disk[:serial_number] = Regexp.last_match(2)
        when "Model" then disk[:model] = Regexp.last_match(2)
        end
      elsif array && line =~ /^         Status:\s+(.*\S)\s*$/
        case Regexp.last_match(1)
        when "OK" then array[:status] = "optimal"
        when "Interim Recovery Mode" then array[:status] = "degraded"
        else array[:status] = "unknown"
        end
      elsif array && line =~ /^         (\S[^:]+):\s+(.*\S)\s*$/
        case Regexp.last_match(1)
        when "Size" then array[:size] = Regexp.last_match(2)
        when "Fault Tolerance" then array[:raid_level] = Regexp.last_match(2)
        when "Status" then array[:status] = Regexp.last_match(2)
        when "Disk Name" then array[:device] = Regexp.last_match(2).strip
        when "Mount Points" then array[:mount_point] = Regexp.last_match(2).split.first
        when "Unique Identifier" then array[:wwn] = Regexp.last_match(2)
        end
      end
    end

    controllers.each do |controller|
      slot = controller[:slot]

      IO.popen(%W(ssacli controller slot=#{slot} pd all show status)).each do |line|
        if line =~ /^   physicaldrive (\S+) /
          disks << Regexp.last_match(1)
        end
      end

      if device = Dir.glob("/sys/bus/pci/devices/#{controller[:pci_slot]}/cciss*").first
        controller[:device] = File.basename(device).sub(/^cciss(\d+)$/, "/dev/cciss/c\\1d0")
      elsif device = Dir.glob("/sys/bus/pci/devices/#{controller[:pci_slot]}/host*/target*:3:0/*:3:0:0/scsi_generic/sg*").first
        controller[:device] = "/dev/#{File.basename(device)}"
      elsif device = Dir.glob("/sys/bus/pci/devices/#{controller[:pci_slot]}/host*/target*:1:0/*:1:0:*/scsi_generic/sg*").first
        controller[:device] = "/dev/#{File.basename(device)}"
      end
    end

    devices[:disks].each do |disk|
      disk[:smart_device] = "cciss,#{disks.find_index(disk[:location])}"

      if disk[:status] == "Failed"
        disk[:status] = "failed"
      elsif disk[:status] == "Predictive Failure"
        disk[:status] = "failed"
      elsif disk[:status] == "OK" && disk[:drive_type] == "Data Drive"
        disk[:status] = "online"
      elsif disk[:status] == "OK" && disk[:drive_type] == "Spare Drive"
        disk[:status] = "hotspare"
      elsif disk[:drive_type] == "Unassigned Drive"
        disk[:status] = "unconfigured"
      else
        disk[:status] = "unknown"
      end

      disk.delete(:drive_type)
    end
  end

  def find_megaraid_disks(devices)
    controllers = []
    arrays = []

    controller = nil
    array = nil
    disk = nil

    IO.popen(%w(megacli -AdpGetPciInfo -aAll -NoLog)).each do |line|
      if line =~ /^PCI information for Controller (\d+)$/
        controller = {
          :id => devices[:controllers].count,
          :type => "megaraid",
          :arrays => [],
          :disks => []
        }

        devices[:controllers] << controller

        controllers << controller
      elsif line =~ /^Bus Number\s+:\s+([0-9a-f]+)$/i
        controller[:pci_slot] = format "0000:%02x", Integer("0x#{Regexp.last_match(1)}")
      elsif line =~ /^Device Number\s+:\s+([0-9a-f]+)$/i
        controller[:pci_slot] = format "%s:%02x", controller[:pci_slot], Integer("0x#{Regexp.last_match(1)}")
      elsif line =~ /^Function Number\s+:\s+([0-9a-f]+)$/i
        controller[:pci_slot] = format "%s.%01x", controller[:pci_slot], Integer("0x#{Regexp.last_match(1)}")
      end
    end

    IO.popen(%w(megacli -AdpAllInfo -aAll -NoLog)).each do |line|
      if line =~ /^Adapter #(\d+)$/
        controller = controllers[Regexp.last_match(1).to_i]
      elsif line =~ /^(\S.*\S)\s*:\s+(\S.*)$/
        case Regexp.last_match(1)
        when "Product Name" then controller[:model] = Regexp.last_match(2)
        when "Serial No" then controller[:serial_number] = Regexp.last_match(2)
        when "FW Package Build" then controller[:firmware_version] = Regexp.last_match(2)
        end
      end
    end

    IO.popen(%w(megacli -LdPdInfo -aAll -NoLog)).each do |line|
      if line =~ /^Adapter #(\d+)$/
        controller = controllers[Regexp.last_match(1).to_i]
      elsif controller && line =~ /^Virtual Drive: (\d+) \(Target Id: (\d+)\)$/
        pci_slot = controller[:pci_slot]
        target = Regexp.last_match(2)
        device = Dir.glob("/sys/bus/pci/devices/#{pci_slot}/host*/target*:2:#{target}/*:2:#{target}:0/block/*").first

        array = {
          :id => devices[:arrays].count,
          :controller => controller[:id],
          :number => Regexp.last_match(1),
          :device => "/dev/#{File.basename(device)}",
          :disks => []
        }

        devices[:arrays] << array
        controller[:arrays] << array[:id]

        arrays << array

        disk = nil
      elsif array && line =~ /^PD: (\d+) Information$/
        disk = {
          :id => devices[:disks].count,
          :controller => controller[:id],
          :arrays => [array[:id]]
        }

        devices[:disks] << disk
        controller[:disks] << disk[:id]
        array[:disks] << disk[:id]
      elsif disk && line =~ /^Firmware state:\s+(\S.*)$/
        status, state = Regexp.last_match(1).split(/,\s*/)
        case status
        when "Unconfigured(good)" then disk[:status] = "unconfigured"
        when "Unconfigured(bad)" then disk[:status] = "unconfigured"
        when "Hotspare" then disk[:status] = "hotspare"
        when "Offline" then disk[:status] = "offline"
        when "Online" then disk[:status] = "online"
        when "Rebuild" then disk[:status] = "rebuilding"
        when "Failed" then disk[:status] = "failed"
        when "Copyback" then disk[:status] = "rebuilding"
        else disk[:status] = "unknown"
        end
        case state
        when "Spun Up" then disk[:state] = "spun_up"
        when "Spun down" then disk[:state] = "spun_down"
        else disk[:state] = "unknown"
        end
      elsif disk && line =~ /^(\S.*\S)\s*:\s+(\S.*)$/
        case Regexp.last_match(1)
        when "Device Id" then disk[:smart_device] = "megaraid,#{Regexp.last_match(2)}"
        when "WWN" then disk[:wwn] = Regexp.last_match(2)
        when "PD Type" then disk[:interface] = Regexp.last_match(2)
        when "Raw Size" then disk[:size] = memory_to_disk_size(Regexp.last_match(2).sub(/\s*\[.*\]$/, ""))
        when "Inquiry Data" then disk[:vendor], disk[:model], disk[:serial_number] = Regexp.last_match(2).split
        end
      elsif array && line =~ /^State\s*:\s+(.*\S)\s*$/
        case Regexp.last_match(1)
        when "Partially Degraded" then array[:status] = "degraded"
        when "Degraded" then array[:status] = "degraded"
        when "Optimal" then array[:status] = "optimal"
        when "Consistency Check" then array[:status] = "checking"
        when "Background Initialization" then array[:status] = "initialising"
        when "Initialization" then array[:status] = "initialising"
        when "Reconstruction" then array[:status] = "rebuilding"
        else array[:status] = "unknown"
        end
      elsif array && line =~ /^(\S.*\S)\s*:\s+(\S.*)$/
        case Regexp.last_match(1)
        when "RAID Level" then array[:raid_level] = Regexp.last_match(2).scan(/Primary-(\d+)/).first.first
        when "Size" then array[:size] = Regexp.last_match(2)
        end
      end
    end

    IO.popen(%w(megacli -PDList -aAll -NoLog)).each do |line|
      if line =~ /^Adapter #(\d+)$/
        controller = controllers[Regexp.last_match(1).to_i]
      elsif controller && line =~ /^Enclosure Device ID: \d+$/
        disk = {
          :controller => controller[:id]
        }
      elsif disk && line =~ /^WWN:\s+(\S+)$/
        unless devices[:disks].find { |d| d[:wwn] == Regexp.last_match(1) }
          disk[:id] = devices[:disks].count
          disk[:wwn] = Regexp.last_match(1)

          devices[:disks] << disk
        end
      elsif disk && line =~ /^Firmware state:\s+(\S.*)$/
        status, state = Regexp.last_match(1).split(/,\s*/)
        case status
        when "Unconfigured(good)" then disk[:status] = "unconfigured"
        when "Unconfigured(bad)" then disk[:status] = "unconfigured"
        when "Hotspare" then disk[:status] = "hotspare"
        when "Offline" then disk[:status] = "offline"
        when "Online" then disk[:status] = "online"
        when "Rebuild" then disk[:status] = "rebuilding"
        when "Failed" then disk[:status] = "failed"
        when "Copyback" then disk[:status] = "rebuilding"
        else disk[:status] = "unknown"
        end
        case state
        when "Spun Up" then disk[:state] = "spun_up"
        when "Spun down" then disk[:state] = "spun_down"
        else disk[:state] = "unknown"
        end
      elsif disk && line =~ /^(\S.*\S)\s*:\s+(\S.*)$/
        case Regexp.last_match(1)
        when "Device Id" then disk[:smart_device] = "megaraid,#{Regexp.last_match(2)}"
        when "PD Type" then disk[:interface] = Regexp.last_match(2)
        when "Raw Size" then disk[:size] = memory_to_disk_size(Regexp.last_match(2).sub(/\s*\[.*\]$/, ""))
        when "Inquiry Data" then disk[:vendor], disk[:model], disk[:serial_number] = Regexp.last_match(2).split
        end
      end
    end

    controllers.each do |controller|
      if device = Dir.glob("/sys/bus/pci/devices/#{controller[:pci_slot]}/host*/target*:2:0/*/scsi_generic/sg*").first
        controller[:device] = "/dev/#{File.basename(device)}"
      end
    end
  end

  def find_mpt1_disks(devices)
    controllers = []
    disks = []

    controller = nil

    IO.popen(%w(lsiutil -s)).each do |line|
      if line =~ /^\/proc\/mpt\/ioc(\d+)\s+LSI Logic\s+(\S+)\s+/
        controller = {
          :id => devices[:controllers].count,
          :type => "mpt1",
          :model => Regexp.last_match(1),
          :arrays => [],
          :disks => []
        }

        controllers << controller
        devices[:controllers] << controller
      elsif line =~ /^\s+(\d+)\s+(\d+)\s+PhysDisk (\d+)\s+(\S+)\s+(\S+)\s+\d+\s+(\S+)\s+/
        disks[Regexp.last_match(3).to_i] = {
          :id => devices[:disks].count,
          :controller => controller[:id],
          :vendor => Regexp.last_match(4),
          :model => Regexp.last_match(5),
          :sas_address => Regexp.last_match(6),<commit_msg>Fix detection of pass through disks on megaraid controllers<commit_after>require "pathname"

Ohai.plugin(:Hardware) do
  provides "hardware"

  def read_sysctl_link(file)
    File.basename(File.readlink(file))
  rescue Errno::ENOENT, Errno::ENOTDIR
  end

  def read_sysctl_file(file)
    IO.read(file).strip
  rescue Errno::ENOENT, Errno::ENOTDIR, Errno::EINVAL
  end

  def parse_memory_size(size)
    if size =~ /^(\d+(?:\.\d+)?)\s*TB/i
      Regexp.last_match(1).to_f * 2**30
    elsif size =~ /^(\d+(?:\.\d+)?)\s*GB/i
      Regexp.last_match(1).to_f * 2**20
    elsif size =~ /^(\d+(?:\.\d+)?)\s*MB/i
      Regexp.last_match(1).to_f * 2**10
    end
  end

  def format_disk_size(kb)
    if kb == 0
      ""
    else
      kblog10 = Math.log10(kb).floor

      kb = kb.to_f * 2 / 10**kblog10
      kb = kb.round.to_f / 2

      if kblog10 >= 9
        format "%gTB", kb * 10**(kblog10 - 9)
      elsif kblog10 >= 6
        format "%dGB", kb * 10**(kblog10 - 6)
      else
        format "%dMB", kb * 10**(kblog10 - 3)
      end
    end
  end

  def memory_to_disk_size(size)
    format_disk_size(parse_memory_size(size))
  end

  def find_sas_device(address)
    file = Dir.glob("/sys/class/scsi_generic/sg*/device/sas_address").find do |file|
      read_sysctl_file(file) == "0x#{address}"
    end

    if file
      dir = File.dirname(file)
      device = Dir.glob("#{dir}/block/*").first ||
               Dir.glob("#{dir}/scsi_generic/*").first

      "/dev/#{File.basename(device)}"
    end
  end

  def pci_devices
    devices = {}
    device = nil

    IO.popen(["lspci", "-Dkvmm"]).each do |line|
      if line =~ /^Slot:\s+((\h{4}):(\h{2}):(\h{2}).(\h))\s*$/
        device = {
          :slot => Regexp.last_match(1),
          :domain => Regexp.last_match(2),
          :bus => Regexp.last_match(3),
          :device => Regexp.last_match(4),
          :function => Regexp.last_match(5)
        }
      elsif device && line =~ /^([A-Z]+):\s+(.*)\s*$/i
        case Regexp.last_match(1)
        when "Class" then device[:class_name] = Regexp.last_match(2)
        when "Vendor" then device[:vendor_name] = Regexp.last_match(2)
        when "Device" then device[:device_name] = Regexp.last_match(2)
        when "SVendor" then device[:subsystem_vendor_name] = Regexp.last_match(2)
        when "SDevice" then device[:subsystem_device_name] = Regexp.last_match(2)
        when "PhySlot" then device[:physical_slot] = Regexp.last_match(2)
        when "Rev" then device[:revision] = Regexp.last_match(2)
        when "ProgIf" then device[:programming_interface] = Regexp.last_match(2)
        when "Driver" then device[:driver] = Regexp.last_match(2)
        when "Module" then device[:modules] = Array(device[:modules]) << Regexp.last_match(2)
        end
      elsif device && line =~ /^\s*$/
        devices[device[:slot]] = device
        device = nil
      end
    end

    IO.popen(["lspci", "-Dkvmmn"]).each do |line|
      if line =~ /^Slot:\s+((\h{4}):(\h{2}):(\h{2}).(\h))\s*$/
        device = devices[Regexp.last_match(1)]
      elsif device && line =~ /^([A-Z]+):\s+(.*)\s*$/i
        case Regexp.last_match(1)
        when "Class" then device[:class_id] = Regexp.last_match(2)
        when "Vendor" then device[:vendor_id] = Regexp.last_match(2)
        when "Device" then device[:device_id] = Regexp.last_match(2)
        when "SVendor" then device[:subsystem_vendor_id] = Regexp.last_match(2)
        when "SDevice" then device[:subsystem_device_id] = Regexp.last_match(2)
        end
      elsif device && line =~ /^\s*$/
        device = nil
      end
    end

    devices
  end

  def network_devices
    Dir.glob("/sys/class/net/*").each_with_object(Mash.new) do |device, devices|
      name = File.basename(device)

      devices[name] = {
        :device => read_sysctl_link("#{device}/device"),
        :duplex => read_sysctl_file("#{device}/duplex"),
        :speed => read_sysctl_file("#{device}/speed")
      }.delete_if { |_, v| v.nil? }
    end
  end

  def memory_devices
    device = nil

    IO.popen(["dmidecode", "-t", "memory"]).each_with_object([]) do |line, devices|
      if line =~ /^Memory Device\s*$/
        device = {}
      elsif device && line =~ /^\s+([A-Z ]+):\s+(.*)\s*$/i
        device[Regexp.last_match(1).tr(" ", "_").downcase.to_sym] = Regexp.last_match(2).strip
      elsif device && line =~ /^\s*$/
        devices << device
        device = nil
      end
    end
  end

  def disk_devices
    disk = Mash.new

    disk[:controllers] = []
    disk[:arrays] = []
    disk[:disks] = []

    find_direct_disks(disk)
    find_nvme_disks(disk)

    find_hp_disks(disk) if File.exist?("/usr/sbin/ssacli")
    find_megaraid_disks(disk) if File.exist?("/usr/sbin/megacli")
    find_mpt1_disks(disk) if File.exist?("/usr/sbin/lsiutil")
    find_mpt2_disks(disk) if File.exist?("/usr/sbin/sas2ircu")
    find_adaptec_disks(disk) if File.exist?("/usr/sbin/arcconf")
    find_areca_disks(disk) if File.exist?("/opt/areca/x86_64/cli64")

    find_md_arrays(disk)

    disk[:disks].each do |disk|
      if disk[:vendor] =~ /^(BTWA|CVPR|PHDV)/ && disk[:model] == "INTEL"
        disk[:model] = disk[:serial_number]
        disk[:serial_number] = disk[:vendor]
        disk[:vendor] = "INTEL"
      end

      if disk[:vendor].nil? && disk[:model] =~ /^ATA\s+(.*)$/
        disk[:vendor] = "ATA"
        disk[:model] = Regexp.last_match(1)
      end

      if disk[:vendor].nil? || disk[:vendor] == "ATA"
        if disk[:model] =~ /^(\S+)\s+(.*)$/
          disk[:vendor] = Regexp.last_match(1)
          disk[:model] = Regexp.last_match(2)
        elsif disk[:model] =~ /^ST/
          disk[:vendor] = "SEAGATE"
        elsif disk[:model] =~ /^C300-(.*)$/
          disk[:vendor] = "CRUCIAL"
          disk[:model] = Regexp.last_match(1)
        end
      end

      disk[:model].sub!(/-.*$/, "") if disk[:model]
    end

    disk
  end

  def find_direct_disks(devices)
    Dir.glob("/sys/class/scsi_host/host*") do |host|
      driver = read_sysctl_file("#{host}/proc_name")

      if %w(ahci mptsas sata_mv sata_nv).include?(driver)
        bus = host.sub("/sys/class/scsi_host/host", "")

        Dir.glob("/sys/bus/scsi/devices/#{bus}:0:*").each do |device|
          next unless File.exist?("#{device}/scsi_disk")

          block = Dir.glob("#{device}/block/*").first
          size = read_sysctl_file("#{block}/size").to_f / 2

          devices[:disks] << {
            :id => devices[:disks].count,
            :device => "/dev/#{File.basename(block)}",
            :vendor => read_sysctl_file("#{device}/vendor"),
            :model => read_sysctl_file("#{device}/model"),
            :firmware_version => read_sysctl_file("#{device}/rev"),
            :size => format_disk_size(size),
            :arrays => []
          }
        end
      end
    end
  end

  def find_nvme_disks(devices)
    Dir.glob("/sys/class/nvme/nvme*") do |device|
      controller = {
        :id => devices[:controllers].count,
        :pci_slot => File.basename(Pathname.new("#{device}/device").realpath),
        :arrays => [],
        :disks => []
      }

      devices[:controllers] << controller

      IO.popen(["lspci", "-Dkvmm", "-s", controller[:pci_slot]]).each do |line|
        if line =~ /^SVendor:\s+(\S.*\S)\s*$/
          controller[:vendor] = Regexp.last_match(1)
        elsif line =~ /^SDevice:\s+(\S.*\S)\s*$/
          controller[:model] = Regexp.last_match(1)
        end
      end

      Dir.glob("#{device}/nvme*").each do |block|
        size = read_sysctl_file("#{block}/size").to_f / 2

        disk = {
          :id => devices[:disks].count,
          :controller => controller[:id],
          :device => "/dev/#{File.basename(block)}",
          :vendor => controller[:vendor],
          :model => controller[:model],
          :size => format_disk_size(size),
          :arrays => []
        }

        devices[:disks] << disk
        controller[:disks] << disk[:id]
      end
    end
  end

  def find_md_arrays(devices)
    array = nil

    File.new("/proc/mdstat", "r").each do |line|
      if line =~ /^(md\d+) : active raid(\d+)((?: (?:sd[a-z]\d*|nvme\d+n\d+(?:p\d+)?)\[\d+\](?:\([A-Z]\))*)+)$/
        array = {
          :id => devices[:arrays].count,
          :device => "/dev/#{Regexp.last_match(1)}",
          :status => "optimal",
          :raid_level => Regexp.last_match(2),
          :disks => []
        }

        Regexp.last_match(3).split(" ").each do |member|
          if member =~ /^(sd[a-z]+|nvme\d+n\d+).*/
            device = Regexp.last_match(1)

            if disk = devices[:disks].find { |d| d[:device] == "/dev/#{device}" }
              if member =~ /\(F\)/
                disk[:status] = "failed"
              elsif member =~ /\(S\)/
                disk[:status] = "hotspare"
              else
                disk[:status] = "online"
              end

              disk[:arrays] << array[:id]
              array[:disks] << disk[:id]
            end
          end
        end

        devices[:arrays] << array
      elsif array && line =~ /^\s+(\d+) blocks.*(?:\[([U_]+)\])?/
        array[:size] = format_disk_size(Regexp.last_match(1).to_i)
        array[:status] = "degraded" if Regexp.last_match(2) =~ /_/
      elsif array && line =~ /^\s+\[.*\]\s+(\S+)\s+=/
        case Regexp.last_match(1)
        when "recovery" then array[:status] = "rebuilding"
        when "resync" then array[:status] = "rebuilding"
        when "checking" then array[:status] = "checking"
        end
      end
    end
  end

  def find_hp_disks(devices)
    controllers = []
    disks = []

    controller = nil
    array = nil
    disk = nil

    IO.popen(%w(ssacli controller all show config detail)).each do |line|
      next unless line.valid_encoding?

      if line =~ /^Smart (?:Array|HBA) (\S+) /
        controller = {
          :id => devices[:controllers].count,
          :type => "hp",
          :model => Regexp.last_match(1),
          :arrays => [],
          :disks => []
        }

        devices[:controllers] << controller

        controllers << controller

        array = nil
        disk = nil
      elsif controller && line =~ /^   (\S.*):\s+(.*)$/
        case Regexp.last_match(1)
        when "Slot" then controller[:slot] = Regexp.last_match(2)
        when "Serial Number" then controller[:serial_number] = Regexp.last_match(2)
        when "Hardware Revision" then controller[:hardware_version] = Regexp.last_match(2)
        when "Firmware Version" then controller[:firmware_version] = Regexp.last_match(2)
        when "PCI Address (Domain:Bus:Device.Function)" then controller[:pci_slot] = Regexp.last_match(2)
        end
      elsif controller && line =~ /^      Logical Drive: (\d+)$/
        array = {
          :id => devices[:arrays].count,
          :controller => controller[:id],
          :number => Regexp.last_match(1),
          :disks => []
        }

        devices[:arrays] << array
        controller[:arrays] << array[:id]

        disk = nil
      elsif array && line =~ /^      physicaldrive (\S+)$/
        disk = {
          :id => devices[:disks].count,
          :controller => controller[:id],
          :arrays => [array[:id]],
          :location => Regexp.last_match(1)
        }

        devices[:disks] << disk
        controller[:disks] << disk[:id]
        array[:disks] << disk[:id]
      elsif disk && line =~ /^         (\S[^:]+):\s+(.*\S)\s*$/
        case Regexp.last_match(1)
        when "Status" then disk[:status] = Regexp.last_match(2)
        when "Drive Type" then disk[:drive_type] = Regexp.last_match(2)
        when "Interface Type" then disk[:interface] = Regexp.last_match(2)
        when "Size" then disk[:size] = Regexp.last_match(2)
        when "Rotational Speed" then disk[:rpm] = Regexp.last_match(2)
        when "Firmware Revision" then disk[:firmware_version] = Regexp.last_match(2)
        when "Serial Number" then disk[:serial_number] = Regexp.last_match(2)
        when "Model" then disk[:model] = Regexp.last_match(2)
        end
      elsif array && line =~ /^         Status:\s+(.*\S)\s*$/
        case Regexp.last_match(1)
        when "OK" then array[:status] = "optimal"
        when "Interim Recovery Mode" then array[:status] = "degraded"
        else array[:status] = "unknown"
        end
      elsif array && line =~ /^         (\S[^:]+):\s+(.*\S)\s*$/
        case Regexp.last_match(1)
        when "Size" then array[:size] = Regexp.last_match(2)
        when "Fault Tolerance" then array[:raid_level] = Regexp.last_match(2)
        when "Status" then array[:status] = Regexp.last_match(2)
        when "Disk Name" then array[:device] = Regexp.last_match(2).strip
        when "Mount Points" then array[:mount_point] = Regexp.last_match(2).split.first
        when "Unique Identifier" then array[:wwn] = Regexp.last_match(2)
        end
      end
    end

    controllers.each do |controller|
      slot = controller[:slot]

      IO.popen(%W(ssacli controller slot=#{slot} pd all show status)).each do |line|
        if line =~ /^   physicaldrive (\S+) /
          disks << Regexp.last_match(1)
        end
      end

      if device = Dir.glob("/sys/bus/pci/devices/#{controller[:pci_slot]}/cciss*").first
        controller[:device] = File.basename(device).sub(/^cciss(\d+)$/, "/dev/cciss/c\\1d0")
      elsif device = Dir.glob("/sys/bus/pci/devices/#{controller[:pci_slot]}/host*/target*:3:0/*:3:0:0/scsi_generic/sg*").first
        controller[:device] = "/dev/#{File.basename(device)}"
      elsif device = Dir.glob("/sys/bus/pci/devices/#{controller[:pci_slot]}/host*/target*:1:0/*:1:0:*/scsi_generic/sg*").first
        controller[:device] = "/dev/#{File.basename(device)}"
      end
    end

    devices[:disks].each do |disk|
      disk[:smart_device] = "cciss,#{disks.find_index(disk[:location])}"

      if disk[:status] == "Failed"
        disk[:status] = "failed"
      elsif disk[:status] == "Predictive Failure"
        disk[:status] = "failed"
      elsif disk[:status] == "OK" && disk[:drive_type] == "Data Drive"
        disk[:status] = "online"
      elsif disk[:status] == "OK" && disk[:drive_type] == "Spare Drive"
        disk[:status] = "hotspare"
      elsif disk[:drive_type] == "Unassigned Drive"
        disk[:status] = "unconfigured"
      else
        disk[:status] = "unknown"
      end

      disk.delete(:drive_type)
    end
  end

  def find_megaraid_disks(devices)
    controllers = []
    arrays = []
    disks = []

    controller = nil
    array = nil
    disk = nil

    IO.popen(%w(megacli -AdpGetPciInfo -aAll -NoLog)).each do |line|
      if line =~ /^PCI information for Controller (\d+)$/
        controller = {
          :id => devices[:controllers].count,
          :type => "megaraid",
          :arrays => [],
          :disks => []
        }

        devices[:controllers] << controller

        controllers << controller
      elsif line =~ /^Bus Number\s+:\s+([0-9a-f]+)$/i
        controller[:pci_slot] = format "0000:%02x", Integer("0x#{Regexp.last_match(1)}")
      elsif line =~ /^Device Number\s+:\s+([0-9a-f]+)$/i
        controller[:pci_slot] = format "%s:%02x", controller[:pci_slot], Integer("0x#{Regexp.last_match(1)}")
      elsif line =~ /^Function Number\s+:\s+([0-9a-f]+)$/i
        controller[:pci_slot] = format "%s.%01x", controller[:pci_slot], Integer("0x#{Regexp.last_match(1)}")
      end
    end

    IO.popen(%w(megacli -AdpAllInfo -aAll -NoLog)).each do |line|
      if line =~ /^Adapter #(\d+)$/
        controller = controllers[Regexp.last_match(1).to_i]
      elsif line =~ /^(\S.*\S)\s*:\s+(\S.*)$/
        case Regexp.last_match(1)
        when "Product Name" then controller[:model] = Regexp.last_match(2)
        when "Serial No" then controller[:serial_number] = Regexp.last_match(2)
        when "FW Package Build" then controller[:firmware_version] = Regexp.last_match(2)
        end
      end
    end

    IO.popen(%w(megacli -LdPdInfo -aAll -NoLog)).each do |line|
      if line =~ /^Adapter #(\d+)$/
        controller = controllers[Regexp.last_match(1).to_i]
      elsif controller && line =~ /^Virtual Drive: (\d+) \(Target Id: (\d+)\)$/
        pci_slot = controller[:pci_slot]
        target = Regexp.last_match(2)
        device = Dir.glob("/sys/bus/pci/devices/#{pci_slot}/host*/target*:2:#{target}/*:2:#{target}:0/block/*").first

        array = {
          :id => devices[:arrays].count,
          :controller => controller[:id],
          :number => Regexp.last_match(1),
          :device => "/dev/#{File.basename(device)}",
          :disks => []
        }

        devices[:arrays] << array
        controller[:arrays] << array[:id]

        arrays << array

        disk = nil
      elsif array && line =~ /^PD: (\d+) Information$/
        disk = {
          :id => devices[:disks].count,
          :controller => controller[:id],
          :arrays => [array[:id]]
        }

        devices[:disks] << disk
        controller[:disks] << disk[:id]
        array[:disks] << disk[:id]

        disks << disk
      elsif disk && line =~ /^Firmware state:\s+(\S.*)$/
        status, state = Regexp.last_match(1).split(/,\s*/)
        case status
        when "Unconfigured(good)" then disk[:status] = "unconfigured"
        when "Unconfigured(bad)" then disk[:status] = "unconfigured"
        when "Hotspare" then disk[:status] = "hotspare"
        when "Offline" then disk[:status] = "offline"
        when "Online" then disk[:status] = "online"
        when "Rebuild" then disk[:status] = "rebuilding"
        when "Failed" then disk[:status] = "failed"
        when "Copyback" then disk[:status] = "rebuilding"
        else disk[:status] = "unknown"
        end
        case state
        when "Spun Up" then disk[:state] = "spun_up"
        when "Spun down" then disk[:state] = "spun_down"
        else disk[:state] = "unknown"
        end
      elsif disk && line =~ /^(\S.*\S)\s*:\s+(\S.*)$/
        case Regexp.last_match(1)
        when "Device Id" then disk[:smart_device] = "megaraid,#{Regexp.last_match(2)}"
        when "WWN" then disk[:wwn] = Regexp.last_match(2)
        when "PD Type" then disk[:interface] = Regexp.last_match(2)
        when "Raw Size" then disk[:size] = memory_to_disk_size(Regexp.last_match(2).sub(/\s*\[.*\]$/, ""))
        when "Inquiry Data" then disk[:vendor], disk[:model], disk[:serial_number] = Regexp.last_match(2).split
        end
      elsif array && line =~ /^State\s*:\s+(.*\S)\s*$/
        case Regexp.last_match(1)
        when "Partially Degraded" then array[:status] = "degraded"
        when "Degraded" then array[:status] = "degraded"
        when "Optimal" then array[:status] = "optimal"
        when "Consistency Check" then array[:status] = "checking"
        when "Background Initialization" then array[:status] = "initialising"
        when "Initialization" then array[:status] = "initialising"
        when "Reconstruction" then array[:status] = "rebuilding"
        else array[:status] = "unknown"
        end
      elsif array && line =~ /^(\S.*\S)\s*:\s+(\S.*)$/
        case Regexp.last_match(1)
        when "RAID Level" then array[:raid_level] = Regexp.last_match(2).scan(/Primary-(\d+)/).first.first
        when "Size" then array[:size] = Regexp.last_match(2)
        end
      end
    end

    IO.popen(%w(megacli -PDList -aAll -NoLog)).each do |line|
      if line =~ /^Adapter #(\d+)$/
        controller = controllers[Regexp.last_match(1).to_i]
      elsif controller && line =~ /^Enclosure Device ID: \d+$/
        disk = {
          :controller => controller[:id],
          :arrays => []
        }
      elsif disk && line =~ /^WWN:\s+(\S+)$/
        unless devices[:disks].find { |d| d[:wwn] == Regexp.last_match(1) }
          disk[:id] = devices[:disks].count
          disk[:wwn] = Regexp.last_match(1)

          devices[:disks] << disk

          disks << disk
        end
      elsif disk && line =~ /^Firmware state:\s+(\S.*)$/
        status, state = Regexp.last_match(1).split(/,\s*/)
        case status
        when "Unconfigured(good)" then disk[:status] = "unconfigured"
        when "Unconfigured(bad)" then disk[:status] = "unconfigured"
        when "Hotspare" then disk[:status] = "hotspare"
        when "Offline" then disk[:status] = "offline"
        when "Online" then disk[:status] = "online"
        when "Rebuild" then disk[:status] = "rebuilding"
        when "Failed" then disk[:status] = "failed"
        when "Copyback" then disk[:status] = "rebuilding"
        else disk[:status] = "unknown"
        end
        case state
        when "Spun Up" then disk[:state] = "spun_up"
        when "Spun down" then disk[:state] = "spun_down"
        else disk[:state] = "unknown"
        end
      elsif disk && line =~ /^(\S.*\S)\s*:\s+(\S.*)$/
        case Regexp.last_match(1)
        when "Device Id" then disk[:device_id] = Regexp.last_match(2)
        when "PD Type" then disk[:interface] = Regexp.last_match(2)
        when "Raw Size" then disk[:size] = memory_to_disk_size(Regexp.last_match(2).sub(/\s*\[.*\]$/, ""))
        when "Inquiry Data" then disk[:vendor], disk[:model], disk[:serial_number] = Regexp.last_match(2).split
        end
      end
    end

    controllers.each do |controller|
      if device = Dir.glob("/sys/bus/pci/devices/#{controller[:pci_slot]}/host*/target*:2:0/*/scsi_generic/sg*").first
        controller[:device] = "/dev/#{File.basename(device)}"
      end
    end

    disks.each do |disk|
      controller = devices[:controllers][disk[:controller]]
      id = disk.delete(:device_id)

      if device = Dir.glob("/sys/bus/pci/devices/#{controller[:pci_slot]}/host*/target0:0:#{id}/0:0:#{id}:0/block/sd*").first
        disk[:device] = "/dev/#{File.basename(device)}"
      else
        disk[:smart_device] = "megaraid,#{id}"
      end
    end
  end

  def find_mpt1_disks(devices)
    controllers = []
    disks = []

    controller = nil

    IO.popen(%w(lsiutil -s)).each do |line|
      if line =~ /^\/proc\/mpt\/ioc(\d+)\s+LSI Logic\s+(\S+)\s+/
        controller = {
          :id => devices[:controllers].count,
          :type => "mpt1",
          :model => Regexp.last_match(1),
          :arrays => [],
          :disks => []
        }

        controllers << controller
        devices[:controllers] << controller
      elsif line =~ /^\s+(\d+)\s+(\d+)\s+PhysDisk (\d+)\s+(\S+)\s+(\S+)\s+\d+\s+(\S+)\s+/
        disks[Regexp.last_match(3).to_i] = {
          :id => devices[:disks].count,
          :controller => controller[:id],
          :vendor => Regexp.last_match(4),
          :model => Regexp.last_match(5),
          :sas_address => Regexp.last_match(6),
{'fraction_non_alphanumeric': 0.1799405413863245, 'fraction_numerical': 0.009231732123298388, 'mean_word_length': 2.3582044738027323, 'pattern_counts': {'":': 0, '<': 119, '<?xml version=': 0, '>': 142, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6953144', 'n_tokens_mistral': 15842, 'n_tokens_neox': 15372, 'n_words': 2920}