<commit_before>require 'datacite/mapping'

module Datacite
  module Mapping

    class Rights
      # TODO: get real URL
      UCSF_DUA = Rights.new(
        uri: URI('https://datashare.ucsf.edu/xtf/search?smode=dataUseAgreementUCSF'),
        value: 'UCSF Datashare Data Use Agreement'
      )
      # TODO: get real URL
      UCSF_FEB_13 = Rights.new(
        uri:  URI('https://merritt.cdlib.org/d/ark%3A%2Fb7272%2Fq6bg2kwf/6/producer%2FDUA_formal_BMJopen_female%20condomt.docx'),
        value: 'custom Data Use Agreement'
      )
    end

    class Description
      # TODO: consdier pushing this to datacite-mapping
      def value=(v)
        @value = v.strip.squeeze(' ').gsub(/-[ \n]+/, '')
      end
    end

    class Rights
      # TODO: consdier pushing this to datacite-mapping
      def value=(v)
        @value = v.strip.tr("\n", ' ').squeeze(' ')
      end
    end

    class Identifier
      # TODO: consdier pushing this to datacite-mapping
      def value=(v)
        v = v.strip.upcase
        raise ArgumentError, 'Identifier must have a non-nil value' unless v
        raise ArgumentError, "Identifier value '#{v}' is not a valid DOI" unless v =~ %r{10\..+/.+}
        @value = v
      end
    end

    class FundingReference
      def to_description
        article = 'the ' unless name.downcase.start_with?('the') || name.start_with?('<NAME>')
        desc_text = "Data were created with funding from #{article}#{name}#{grant_info}."
        Description.new(type: DescriptionType::OTHER, value: desc_text)
      end

      def grant_number
        award_number && award_number.value
      end

      def grant_info
        grant_number &&
          if grant_number.include?('and')
            " under grants #{grant_number}"
          elsif grant_number.downcase.include?('grant') || grant_number.downcase.include?('agreement')
            " under #{grant_number}"
          else
            " under grant #{grant_number}"
          end
      end
    end

    class Resource

      SPECIAL_CASES = {
        %r{<(identifier|subject)[^>]+/>} => '', # remove empty tags
        %r{<(identifier|subject)[^>]+>\s+</\1>} => '', # remove empty tag pairs
        %r{(<date[^>]*>)(\d{4})-(\d{4})(</date>)} => '\\1\\2/\\3\\4', # fix date ranges
        %r{(<contributor[^>/]+>\s*)<contributor>([^<]+)</contributor>(\s*</contributor>)} => '\\1<contributorName>\\2</contributorName>\\3', # fix broken contributors
        'Affaits, National Institutes of Health,' => 'Affairs; National Institutes of Health;',
        'NIH RO1 HL31113, VA BX001970' => 'VA BX001970; NIH RO1 HL31113; nil', # the string 'nil' is special in add_funding_reference()
        'Funding for the preparation of this data was supported by the Bill &amp; Melinda Gates Foundation. The original data collection was supported by grants from the MacArthur Foundation, National Institutes of Health, and the Bill &amp; Melinda Gates Foundation.' =>
          'Bill &amp; Melinda Gates Foundation; MacArthur Foundation; National Institutes of Health; Bill &amp; Melinda Gates Foundation',
        'Current dataset preparation: Bill and Melinda Gates Foundation (OPP1086183). Original data collection: MacArthur Foundation (05-84956-000-GSS), National Institutes of Health (R01HD053129) and Bill and Melinda Gates Foundation (48541).' => 'OPP1086183; 05-84956-000-GSS; R01HD053129; 48541',
        'National Institute of Health' => 'National Institutes of Health',
        'National Geographic Society and National Eye Institute, NIH, US.' => 'National Geographic Society; National Institutes of Health, National Eye Institute',
        'NGS W104-10 and NIH EY022087' => 'NGS W104-10; NIH EY022087',
        'National Institutes of Health and National Science Foundation' => 'National Institutes of Health; National Science Foundation',
        '1R01GM108889-01 (NIH), CHE 1352608 and CHE-0840513 (NSF)' => '1R01GM108889-01; CHE 1352608 and CHE-0840513',
        'National Science Foundation. Office' => 'National Science Foundation, Office',
        'National Science Foundation. Division' => 'National Science Foundation, Division',
        'National Institutes of Health. National' => 'National Institutes of Health, National',
        '. Select a sub-organization' => '',
        'US Bureau of Reclamation Cooperative Agreement' => 'Cooperative Agreement',
        '<description descriptionType="Other"/>' => '',
        '<description descriptionType="Other">0</description>' => '',
        'rightsURI="http:' => 'rightsURI="https:',
        'https://creativecommons.org/about/cc0' => Rights::CC_ZERO.uri.to_s,
        'These data are covered by a Creative Commons CC0 license.' => Rights::CC_ZERO.value,
        'Creative Commons Public Domain Dedication (CC0)' => Rights::CC_ZERO.value,
        'Creative Commons Attribution 4.0 License' => Rights::CC_BY.value,
        'Creative Commons Attribution 4.0 International (CC-BY 4.0)' => Rights::CC_BY.value,
        '<rights>Terms of Use for these data are outlined in the associated Data Use Agreement</rights>' =>
          "<rights rightsURI=\"#{Rights::UCSF_FEB_13.uri}\">#{Rights::UCSF_FEB_13.value}</rights>",
        '<geoLocationPlace>false</geoLocationPlace>' => ''
      }.freeze

      def self.parse_mrt_datacite(mrt_datacite_xml, doi_value)
        resource = parse_special(mrt_datacite_xml)
        resource.ensure_doi(doi_value)
        resource.ensure_resource_type!
        resource.convert_funding!
        resource.fix_breaks!
        resource.inject_rights!
        resource<commit_msg>Inject place points / boxes where needed<commit_after>require 'datacite/mapping'

module Datacite
  module Mapping

    class Rights
      # TODO: get real URL
      UCSF_DUA = Rights.new(
        uri: URI('https://datashare.ucsf.edu/xtf/search?smode=dataUseAgreementUCSF'),
        value: 'UCSF Datashare Data Use Agreement'
      )
      # TODO: get real URL
      UCSF_FEB_13 = Rights.new(
        uri:  URI('https://merritt.cdlib.org/d/ark%3A%2Fb7272%2Fq6bg2kwf/6/producer%2FDUA_formal_BMJopen_female%20condomt.docx'),
        value: 'custom Data Use Agreement'
      )
    end

    class Description
      # TODO: consdier pushing this to datacite-mapping
      def value=(v)
        @value = v.strip.squeeze(' ').gsub(/-[ \n]+/, '')
      end
    end

    class Rights
      # TODO: consdier pushing this to datacite-mapping
      def value=(v)
        @value = v.strip.tr("\n", ' ').squeeze(' ')
      end
    end

    class Identifier
      # TODO: consdier pushing this to datacite-mapping
      def value=(v)
        v = v.strip.upcase
        raise ArgumentError, 'Identifier must have a non-nil value' unless v
        raise ArgumentError, "Identifier value '#{v}' is not a valid DOI" unless v =~ %r{10\..+/.+}
        @value = v
      end
    end

    class FundingReference
      def to_description
        article = 'the ' unless name.downcase.start_with?('the') || name.start_with?('<NAME>')
        desc_text = "Data were created with funding from #{article}#{name}#{grant_info}."
        Description.new(type: DescriptionType::OTHER, value: desc_text)
      end

      def grant_number
        award_number && award_number.value
      end

      def grant_info
        grant_number &&
          if grant_number.include?('and')
            " under grants #{grant_number}"
          elsif grant_number.downcase.include?('grant') || grant_number.downcase.include?('agreement')
            " under #{grant_number}"
          else
            " under grant #{grant_number}"
          end
      end
    end

    class Resource

      SPECIAL_CASES = {
        %r{<(identifier|subject|description)[^>]+/>} => '', # remove empty tags
        %r{<(identifier|subject)[^>]+>\s+</\1>} => '', # remove empty tag pairs
        %r{(<date[^>]*>)(\d{4})-(\d{4})(</date>)} => '\\1\\2/\\3\\4', # fix date ranges
        %r{(<contributor[^>/]+>\s*)<contributor>([^<]+)</contributor>(\s*</contributor>)} => '\\1<contributorName>\\2</contributorName>\\3', # fix broken contributors
        'Affaits, National Institutes of Health,' => 'Affairs; National Institutes of Health;',
        'NIH RO1 HL31113, VA BX001970' => 'VA BX001970; NIH RO1 HL31113; nil', # the string 'nil' is special in add_funding_reference()
        'Funding for the preparation of this data was supported by the Bill &amp; Melinda Gates Foundation. The original data collection was supported by grants from the MacArthur Foundation, National Institutes of Health, and the Bill &amp; Melinda Gates Foundation.' =>
          'Bill &amp; Melinda Gates Foundation; MacArthur Foundation; National Institutes of Health; Bill &amp; Melinda Gates Foundation',
        'Current dataset preparation: Bill and Melinda Gates Foundation (OPP1086183). Original data collection: MacArthur Foundation (05-84956-000-GSS), National Institutes of Health (R01HD053129) and Bill and Melinda Gates Foundation (48541).' => 'OPP1086183; 05-84956-000-GSS; R01HD053129; 48541',
        'National Institute of Health' => 'National Institutes of Health',
        'National Geographic Society and National Eye Institute, NIH, US.' => 'National Geographic Society; National Institutes of Health, National Eye Institute',
        'NGS W104-10 and NIH EY022087' => 'NGS W104-10; NIH EY022087',
        'National Institutes of Health and National Science Foundation' => 'National Institutes of Health; National Science Foundation',
        '1R01GM108889-01 (NIH), CHE 1352608 and CHE-0840513 (NSF)' => '1R01GM108889-01; CHE 1352608 and CHE-0840513',
        'National Science Foundation. Office' => 'National Science Foundation, Office',
        'National Science Foundation. Division' => 'National Science Foundation, Division',
        'National Institutes of Health. National' => 'National Institutes of Health, National',
        '. Select a sub-organization' => '',
        'US Bureau of Reclamation Cooperative Agreement' => 'Cooperative Agreement',
        '<description descriptionType="Other">0</description>' => '',
        'rightsURI="http:' => 'rightsURI="https:',
        'https://creativecommons.org/about/cc0' => Rights::CC_ZERO.uri.to_s,
        'These data are covered by a Creative Commons CC0 license.' => Rights::CC_ZERO.value,
        'Creative Commons Public Domain Dedication (CC0)' => Rights::CC_ZERO.value,
        'Creative Commons Attribution 4.0 License' => Rights::CC_BY.value,
        'Creative Commons Attribution 4.0 International (CC-BY 4.0)' => Rights::CC_BY.value,
        '<rights>Terms of Use for these data are outlined in the associated Data Use Agreement</rights>' => "<rights rightsURI=\"#{Rights::UCSF_FEB_13.uri}\">#{Rights::UCSF_FEB_13.value}</rights>",
        '<geoLocationPlace>false</geoLocationPlace>' => '',
        %r{<geoLocationPlace>Orange County (Calif.)</geoLocationPlace/>} => "\1\n      <geoLocationBox>33.3869 -118.1174 33.9473 -117.4127</geoLocationBox>\n      <geoLocationPoint>33.6671 -117.7651</geoLocationPoint>",
        %r{<geoLocationPlace>Providence Creek (Lower, Upper and P301)</geoLocationPlace>} => "\1\n      <geoLocationPoint>37.047756 -119.221094</geoLocationPoint>"
      }.freeze

      def self.parse_mrt_datacite(mrt_datacite_xml, doi_value)
        resource = parse_special(mrt_datacite_xml)
        resource.ensure_doi(doi_value)
        resource.ensure_resource_type!
        resource.convert_funding!
        resource.fix_breaks!
        resource.inject_rights!
        resource
{'fraction_non_alphanumeric': 0.12799440461619163, 'fraction_numerical': 0.0397796817625459, 'mean_word_length': 3.116228859301907, 'pattern_counts': {'":': 0, '<': 56, '<?xml version=': 0, '>': 119, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3006707', 'n_tokens_mistral': 3771, 'n_tokens_neox': 3365, 'n_words': 937}