<commit_before># wrappers and constants for expressing iNat records as Darwin Core fields
class DarwinCore
  
  DARWIN_CORE_TERMS = [
    %w(id id),
    %w(basisOfRecord http://rs.tdwg.org/dwc/terms/basisOfRecord HumanObservation),
    %w(modified http://purl.org/dc/terms/modified),
    %w(institutionCode http://rs.tdwg.org/dwc/terms/institutionCode iNaturalist),
    %w(collectionCode http://rs.tdwg.org/dwc/terms/collectionCode Observations),
    %w(datasetName http://rs.tdwg.org/dwc/terms/datasetName),
    %w(informationWithheld http://rs.tdwg.org/dwc/terms/informationWithheld),
    %w(catalogNumber http://rs.tdwg.org/dwc/terms/catalogNumber),
    %w(references http://purl.org/dc/terms/references),
    %w(occurrenceRemarks http://rs.tdwg.org/dwc/terms/occurrenceRemarks),
    %w(recordedBy http://rs.tdwg.org/dwc/terms/recordedBy),
    %w(establishmentMeans http://rs.tdwg.org/dwc/terms/establishmentMeans),
    %w(associatedMedia http://rs.tdwg.org/dwc/terms/associatedMedia),
    %w(eventDate http://rs.tdwg.org/dwc/terms/eventDate),
    %w(eventTime http://rs.tdwg.org/dwc/terms/eventTime),
    %w(verbatimEventDate http://rs.tdwg.org/dwc/terms/verbatimEventDate),
    %w(verbatimLocality http://rs.tdwg.org/dwc/terms/verbatimLocality),
    %w(decimalLatitude http://rs.tdwg.org/dwc/terms/decimalLatitude),
    %w(decimalLongitude http://rs.tdwg.org/dwc/terms/decimalLongitude),
    %w(coordinateUncertaintyInMeters http://rs.tdwg.org/dwc/terms/coordinateUncertaintyInMeters),
    %w(identificationID http://rs.tdwg.org/dwc/terms/identificationID),
    %w(dateIdentified http://rs.tdwg.org/dwc/terms/dateIdentified),
    %w(identificationRemarks http://rs.tdwg.org/dwc/terms/identificationRemarks),
    %w(taxonID http://rs.tdwg.org/dwc/terms/taxonID),
    %w(scientificName http://rs.tdwg.org/dwc/terms/scientificName),
    %w(taxonRank http://rs.tdwg.org/dwc/terms/taxonRank),
    %w(rights http://purl.org/dc/terms/rights),
    %w(rightsHolder http://purl.org/dc/terms/rightsHolder)
  ]
  DARWIN_CORE_TERM_NAMES = DARWIN_CORE_TERMS.map{|name, uri| name}
  
  # Extend observation with DwC methods.  For reasons unclear to me, url
  # methods are protected if you instantiate a view *outside* a model, but not
  # inside.  Otherwise I would just used a more traditional adapter with
  # delegation.
  def self.adapt(observation, options = {})
    observation.extend(InstanceMethods)
    observation.set_view(options[:view])
    observation
  end
  
  module InstanceMethods
    def view
      @view ||= FakeView.new
    end

    def set_view(view)
      @view = view
    end

    def occurrenceID
      view.observation_url(self)
    end

    def references
      view.observation_url(self)
    end

    def basisOfRecord
      "HumanObservation"
    end

    def modified
      updated_at.iso8601
    end

    def institutionCode
      "iNaturalist"
    end
    
    def collectionCode
      "Observations"
    end

    def datasetName
      if quality_grade == Observation::RESEARCH_GRADE
        "iNaturalist research-grade observations"
      else
        "iNaturalist observations"
      end
    end

    def informationWithheld
      if geoprivacy_private?
        "Coordinates hidden at the request of the observer"
      elsif geoprivacy_obscured?
        "Coordinate uncertainty increased by #{Observation::M_TO_OBSCURE_THREATENED_TAXA}m at the request of the observer"
      elsif coordinates_obscured?
        "Coordinate uncertainty increased by #{Observation::M_TO_OBSCURE_THREATENED_TAXA}m to protect threatened taxon"
      else
        nil
      end
    end

    def catalogNumber
      id
    end

    def occurrenceRemarks
      dwc_filter_text(description) unless description.blank?
    end


    def recordedBy
      user.name.blank? ? user.login : user.name
    end

    def establishmentMeans
      score = quality_metric_score(QualityMetric::WILD)
      score && score < 0.5 ? "cultivated" : "wild"
    end

    def associatedMedia
      photo_urls = photos.map{|p| [p.original_url, p.native_page_url]}.flatten.compact
      photo_urls.join(',')
    end

    def eventDate
      time_observed_at ? datetime.iso8601 : observed_on.to_s
    end<commit_msg>Bugfix: added space in DWC-A associatedMedia field.<commit_after># wrappers and constants for expressing iNat records as Darwin Core fields
class DarwinCore
  
  DARWIN_CORE_TERMS = [
    %w(id id),
    %w(basisOfRecord http://rs.tdwg.org/dwc/terms/basisOfRecord HumanObservation),
    %w(modified http://purl.org/dc/terms/modified),
    %w(institutionCode http://rs.tdwg.org/dwc/terms/institutionCode iNaturalist),
    %w(collectionCode http://rs.tdwg.org/dwc/terms/collectionCode Observations),
    %w(datasetName http://rs.tdwg.org/dwc/terms/datasetName),
    %w(informationWithheld http://rs.tdwg.org/dwc/terms/informationWithheld),
    %w(catalogNumber http://rs.tdwg.org/dwc/terms/catalogNumber),
    %w(references http://purl.org/dc/terms/references),
    %w(occurrenceRemarks http://rs.tdwg.org/dwc/terms/occurrenceRemarks),
    %w(recordedBy http://rs.tdwg.org/dwc/terms/recordedBy),
    %w(establishmentMeans http://rs.tdwg.org/dwc/terms/establishmentMeans),
    %w(associatedMedia http://rs.tdwg.org/dwc/terms/associatedMedia),
    %w(eventDate http://rs.tdwg.org/dwc/terms/eventDate),
    %w(eventTime http://rs.tdwg.org/dwc/terms/eventTime),
    %w(verbatimEventDate http://rs.tdwg.org/dwc/terms/verbatimEventDate),
    %w(verbatimLocality http://rs.tdwg.org/dwc/terms/verbatimLocality),
    %w(decimalLatitude http://rs.tdwg.org/dwc/terms/decimalLatitude),
    %w(decimalLongitude http://rs.tdwg.org/dwc/terms/decimalLongitude),
    %w(coordinateUncertaintyInMeters http://rs.tdwg.org/dwc/terms/coordinateUncertaintyInMeters),
    %w(identificationID http://rs.tdwg.org/dwc/terms/identificationID),
    %w(dateIdentified http://rs.tdwg.org/dwc/terms/dateIdentified),
    %w(identificationRemarks http://rs.tdwg.org/dwc/terms/identificationRemarks),
    %w(taxonID http://rs.tdwg.org/dwc/terms/taxonID),
    %w(scientificName http://rs.tdwg.org/dwc/terms/scientificName),
    %w(taxonRank http://rs.tdwg.org/dwc/terms/taxonRank),
    %w(rights http://purl.org/dc/terms/rights),
    %w(rightsHolder http://purl.org/dc/terms/rightsHolder)
  ]
  DARWIN_CORE_TERM_NAMES = DARWIN_CORE_TERMS.map{|name, uri| name}
  
  # Extend observation with DwC methods.  For reasons unclear to me, url
  # methods are protected if you instantiate a view *outside* a model, but not
  # inside.  Otherwise I would just used a more traditional adapter with
  # delegation.
  def self.adapt(observation, options = {})
    observation.extend(InstanceMethods)
    observation.set_view(options[:view])
    observation
  end
  
  module InstanceMethods
    def view
      @view ||= FakeView.new
    end

    def set_view(view)
      @view = view
    end

    def occurrenceID
      view.observation_url(self)
    end

    def references
      view.observation_url(self)
    end

    def basisOfRecord
      "HumanObservation"
    end

    def modified
      updated_at.iso8601
    end

    def institutionCode
      "iNaturalist"
    end
    
    def collectionCode
      "Observations"
    end

    def datasetName
      if quality_grade == Observation::RESEARCH_GRADE
        "iNaturalist research-grade observations"
      else
        "iNaturalist observations"
      end
    end

    def informationWithheld
      if geoprivacy_private?
        "Coordinates hidden at the request of the observer"
      elsif geoprivacy_obscured?
        "Coordinate uncertainty increased by #{Observation::M_TO_OBSCURE_THREATENED_TAXA}m at the request of the observer"
      elsif coordinates_obscured?
        "Coordinate uncertainty increased by #{Observation::M_TO_OBSCURE_THREATENED_TAXA}m to protect threatened taxon"
      else
        nil
      end
    end

    def catalogNumber
      id
    end

    def occurrenceRemarks
      dwc_filter_text(description) unless description.blank?
    end


    def recordedBy
      user.name.blank? ? user.login : user.name
    end

    def establishmentMeans
      score = quality_metric_score(QualityMetric::WILD)
      score && score < 0.5 ? "cultivated" : "wild"
    end

    def associatedMedia
      photo_urls = photos.map{|p| [p.original_url, p.native_page_url]}.flatten.compact
      photo_urls.join(', ')
    end

    def eventDate
      time_observed_at ? datetime.iso8601 : observed_on.to_s
    end
{'fraction_non_alphanumeric': 0.11029324468716609, 'fraction_numerical': 0.0023744509082274726, 'mean_word_length': 3.2848423194303153, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4581406', 'n_tokens_mistral': 3040, 'n_tokens_neox': 2800, 'n_words': 505}