<commit_before># frozen_string_literal: true

module DwcaHunter
  class ResourceMammalDiversityDb < DwcaHunter::Resource
    def initialize(opts = {})
      @command = "mammal-div-db"
      @title = "ASM Mammal Diversity Database"
      @url = "https://mammaldiversity.org/species-account/api.php?q=*"
      @UUID = "94270cdd-5424-4bb1-8324-46ccc5386dc7"
      @download_path = File.join(Dir.tmpdir,
                                 "dwca_hunter",
                                 "mammal-div-db",
                                 "data.json")
      @synonyms = []
      @names = []
      @vernaculars = []
      @extensions = []
      @synonyms_hash = {}
      @vernaculars_hash = {}
      super(opts)
    end

    def download
      DwcaHunter.logger_write(object_id, "Downloading")
      `curl '#{@url}' -H 'User-Agent:' -o #{@download_path}`
    end

    def unpack; end

    def make_dwca
      DwcaHunter.logger_write(object_id, "Extracting data")
      get_names
      generate_dwca
    end

    private

    def get_names
      Dir.chdir(@download_dir)
      collect_names
    end

    def collect_names
      @names_index = {}
      decoder = HTMLEntities.new
      data = File.read(File.join(@download_dir, "data.json"))
      data = JSON.parse(data, symbolize_names: true)
      data[:result].each_with_index do |e, _i|
        e = e[1]
        order = e[:dwc][:order].capitalize
        order = nil if order.match(/incertae/)
        family = e[:dwc][:family].capitalize
        family = nil if family.match(/incertae/)
        genus = e[:dwc][:genus].capitalize
        genus = nil if genus.match(/incertae/)
        name = {
          taxon_id: e[:id],
          kingdom: "Animalia",
          phylum: "Chordata",
          klass: "Mammalia",
          order: order,
          family: family,
          genus: genus,
          name_string: "#{e[:dwc][:scientificName]} " \
          "#{e[:dwc][:scientificNameAuthorship][:species]}".strip,
          rank: e[:dwc][:taxonRank],
          status: e[:dwc][:taxonRank],
          code: "ICZN"
        }
        if e[:dwc][:taxonomicStatus] == "accepted"
          @names << name
        else
          @synonyms << name
        end
        vern = e[:dwc][:vernacularName]<commit_msg>mammal-divdb fix id<commit_after># frozen_string_literal: true

module DwcaHunter
  class ResourceMammalDiversityDb < DwcaHunter::Resource
    def initialize(opts = {})
      @command = "mammal-div-db"
      @title = "ASM Mammal Diversity Database"
      @url = "https://mammaldiversity.org/species-account/api.php?q=*"
      @UUID = "94270cdd-5424-4bb1-8324-46ccc5386dc7"
      @download_path = File.join(Dir.tmpdir,
                                 "dwca_hunter",
                                 "mammal-div-db",
                                 "data.json")
      @synonyms = []
      @names = []
      @vernaculars = []
      @extensions = []
      @synonyms_hash = {}
      @vernaculars_hash = {}
      super(opts)
    end

    def download
      DwcaHunter.logger_write(object_id, "Downloading")
      `curl '#{@url}' -H 'User-Agent:' -o #{@download_path}`
    end

    def unpack; end

    def make_dwca
      DwcaHunter.logger_write(object_id, "Extracting data")
      get_names
      generate_dwca
    end

    private

    def get_names
      Dir.chdir(@download_dir)
      collect_names
    end

    def collect_names
      @names_index = {}
      decoder = HTMLEntities.new
      data = File.read(File.join(@download_dir, "data.json"))
      data = JSON.parse(data, symbolize_names: true)
      data[:result].each_with_index do |e, _i|
        e = e[1]
        order = e[:dwc][:order].capitalize
        order = nil if order.match(/incertae/)
        family = e[:dwc][:family].capitalize
        family = nil if family.match(/incertae/)
        genus = e[:dwc][:genus].capitalize
        genus = nil if genus.match(/incertae/)
        name = {
          taxon_id: e[:internal_id],
          kingdom: "Animalia",
          phylum: "Chordata",
          klass: "Mammalia",
          order: order,
          family: family,
          genus: genus,
          name_string: "#{e[:dwc][:scientificName]} " \
          "#{e[:dwc][:scientificNameAuthorship][:species]}".strip,
          rank: e[:dwc][:taxonRank],
          status: e[:dwc][:taxonRank],
          code: "ICZN"
        }
        if e[:dwc][:taxonomicStatus] == "accepted"
          @names << name
        else
          @synonyms << name
        end
        vern = e[:dwc][:vernacularName]
{'fraction_non_alphanumeric': 0.13263719173516997, 'fraction_numerical': 0.010219951121972895, 'mean_word_length': 1.9657444005270093, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2127227', 'n_tokens_mistral': 1539, 'n_tokens_neox': 1480, 'n_words': 267}