<commit_before>class InventoryItem
  include ActiveModel::Model
  include Comparable

  GOVUK_BASE_URL = 'https://www.gov.uk'

  FIELD_POSITIONS = [ :title, :url, :description, :first_published_date, :last_updated, 
    :organisations, :format, :display_type, :policies, :topics, :mainstream_browse_pages, :document_collections,
    :is_withdrawn, :in_history_mode, :matching_queries, :relevance, :recommendation, :redirect_combine_url, :notes ]

  # fields on this model that can be updated from an InventoryItem created from a more recent search
  UPDATABLE_FIELDS = [ :title, :last_updated, :format, :display_type, :description,
    :is_withdrawn, :in_history_mode, :first_published_date]

  # fields on this model which should be merged with an InventoryItem created from a more recent search
  MERGEABLE_FIELDS = [ :topics, :mainstream_browse_pages, :organisations, :policies, :document_collections, :matching_queries]

  # fields that are transformed into an array
  ARRAY_FIELDS = [ :topics, :mainstream_browse_pages, :organisations, :policies, :document_collections, :matching_queries ]

  BOOLEAN_FIELDS = { :is_withdrawn => 'Withdrawn', :in_history_mode => "History mode" }

  ARRAY_SEPARATOR = '; '

  # extra fields we ask for from rummager when doing the search query
  ADDITIONAL_QUERY_FIELDS = [ :link, :title, :description, :public_timestamp, :format, :display_type,  
    :specialist_sectors, :mainstream_browse_pages, :organisations, :policies, :document_collections ].join(',')

  attr_accessor *FIELD_POSITIONS

  private_class_method :new

  def self.new_from_spreadsheet_row(inventory_id, row)
    ActivityLog.debug "Creating Inventory item from spreadsheet row for #{row[0]}", inventory_id
    params = {}
    row.each_with_index do |data_item, i|
      field_name = FIELD_POSITIONS[i]
      if is_array_field?(field_name)
        params[field_name] = make_array(data_item)
      elsif is_boolean_field?(field_name)
        params[field_name] = make_boolean(field_name, data_item)
      else
        params[field_name] = data_item
      end
    end
    params[:url] = row[1].sub(GOVUK_BASE_URL, '')
    params.except!(nil)     # gets rid of any nil keys caused by adding rows to the right
    InventoryItem.send(:new, params)
  end

  def self.new_from_search_result(doc, query_name)
    item = InventoryItem.send(:new)
    item.populate_from_search_result(doc, query_name)
    item
  end

  def populate_from_search_result(doc, query_name)
    @url = doc['link'].nil? ? "No link for #{doc.inspect}" : doc['link'] 
    @title = extract_field(doc, 'title')
    @description = doc['description']
    @last_updated = doc['public_timestamp'].nil? ? nil : Time.parse(doc['public_timestamp']) 
    @format = extract_field(doc, 'format').humanize
    @display_type = extract_field(doc, 'display_type', 'None')
    @topics = extract_from_array_of_hashes(doc, 'specialist_sectors', 'title', 'slug')
    @mainstream_browse_pages = doc['mainstram_browse_page'] || []
    @organisations = extract_from_array_of_hashes(doc, 'organisations', 'acronym')
    @policies = doc['policies'].nil? ? [] : doc['policies'].sort
    @document_collections = extract_from_array_of_hashes(doc, 'document_collections', 'title', 'slug')
    @is_withdrawn = false
    @in_history_mode = false
    @first_published_date = 'Unknown'
    @matching_queries = [query_name]
    @recommendation = ''
    @redirect_combine_url = ''
    @notes = ''
    @relevance = ''
  end

  def <=>(other)
    @url <=> other.url
  end

  # updates this item with an Inventory Item from a more recent search
  def update_from_other_item(other_item)
    update_updatable_fields(other_item)
    update_mergeable_fields(other_item)
    remove_not_returned
  end

  def mark_as_missing
    unless @notes =~ /Not returned from search as of/
      @notes = "Not returned from search as of #{Time.now.strftime('%Y-%m-%d %H:%M')}; " + @notes.to_s
    end
  end

private
  def remove_not_returned
    @notes.gsub!(/Not returned from search as of .*;\s/, '')
  end
  
  def update_updatable_fields(other_item)
    UPDATABLE_FIELDS.each do |fieldname|
      send(putter_method(fieldname), other_item.send(fieldname))
    end
  end

  def update_mergeable_fields(other_item)
    MERGEABLE_FIELDS.each do |fieldname|
      new_value = merge_arrays(other_item, fieldname)
      delete_matching_query_row_numbers(new_value) if fieldname == :matching_queries
      send(putter_method(fieldname), new_value)
    end<commit_msg>Avoid calling `gsub!` on `nil`<commit_after>class InventoryItem
  include ActiveModel::Model
  include Comparable

  GOVUK_BASE_URL = 'https://www.gov.uk'

  FIELD_POSITIONS = [ :title, :url, :description, :first_published_date, :last_updated, 
    :organisations, :format, :display_type, :policies, :topics, :mainstream_browse_pages, :document_collections,
    :is_withdrawn, :in_history_mode, :matching_queries, :relevance, :recommendation, :redirect_combine_url, :notes ]

  # fields on this model that can be updated from an InventoryItem created from a more recent search
  UPDATABLE_FIELDS = [ :title, :last_updated, :format, :display_type, :description,
    :is_withdrawn, :in_history_mode, :first_published_date]

  # fields on this model which should be merged with an InventoryItem created from a more recent search
  MERGEABLE_FIELDS = [ :topics, :mainstream_browse_pages, :organisations, :policies, :document_collections, :matching_queries]

  # fields that are transformed into an array
  ARRAY_FIELDS = [ :topics, :mainstream_browse_pages, :organisations, :policies, :document_collections, :matching_queries ]

  BOOLEAN_FIELDS = { :is_withdrawn => 'Withdrawn', :in_history_mode => "History mode" }

  ARRAY_SEPARATOR = '; '

  # extra fields we ask for from rummager when doing the search query
  ADDITIONAL_QUERY_FIELDS = [ :link, :title, :description, :public_timestamp, :format, :display_type,  
    :specialist_sectors, :mainstream_browse_pages, :organisations, :policies, :document_collections ].join(',')

  attr_accessor *FIELD_POSITIONS

  private_class_method :new

  def self.new_from_spreadsheet_row(inventory_id, row)
    ActivityLog.debug "Creating Inventory item from spreadsheet row for #{row[0]}", inventory_id
    params = {}
    row.each_with_index do |data_item, i|
      field_name = FIELD_POSITIONS[i]
      if is_array_field?(field_name)
        params[field_name] = make_array(data_item)
      elsif is_boolean_field?(field_name)
        params[field_name] = make_boolean(field_name, data_item)
      else
        params[field_name] = data_item
      end
    end
    params[:url] = row[1].sub(GOVUK_BASE_URL, '')
    params.except!(nil)     # gets rid of any nil keys caused by adding rows to the right
    InventoryItem.send(:new, params)
  end

  def self.new_from_search_result(doc, query_name)
    item = InventoryItem.send(:new)
    item.populate_from_search_result(doc, query_name)
    item
  end

  def populate_from_search_result(doc, query_name)
    @url = doc['link'].nil? ? "No link for #{doc.inspect}" : doc['link'] 
    @title = extract_field(doc, 'title')
    @description = doc['description']
    @last_updated = doc['public_timestamp'].nil? ? nil : Time.parse(doc['public_timestamp']) 
    @format = extract_field(doc, 'format').humanize
    @display_type = extract_field(doc, 'display_type', 'None')
    @topics = extract_from_array_of_hashes(doc, 'specialist_sectors', 'title', 'slug')
    @mainstream_browse_pages = doc['mainstram_browse_page'] || []
    @organisations = extract_from_array_of_hashes(doc, 'organisations', 'acronym')
    @policies = doc['policies'].nil? ? [] : doc['policies'].sort
    @document_collections = extract_from_array_of_hashes(doc, 'document_collections', 'title', 'slug')
    @is_withdrawn = false
    @in_history_mode = false
    @first_published_date = 'Unknown'
    @matching_queries = [query_name]
    @recommendation = ''
    @redirect_combine_url = ''
    @notes = ''
    @relevance = ''
  end

  def <=>(other)
    @url <=> other.url
  end

  # updates this item with an Inventory Item from a more recent search
  def update_from_other_item(other_item)
    update_updatable_fields(other_item)
    update_mergeable_fields(other_item)
    remove_not_returned
  end

  def mark_as_missing
    unless @notes =~ /Not returned from search as of/
      @notes = "Not returned from search as of #{Time.now.strftime('%Y-%m-%d %H:%M')}; " + @notes.to_s
    end
  end

private
  def remove_not_returned
    @notes.gsub!(/Not returned from search as of .*;\s/, '') if @notes
  end
  
  def update_updatable_fields(other_item)
    UPDATABLE_FIELDS.each do |fieldname|
      send(putter_method(fieldname), other_item.send(fieldname))
    end
  end

  def update_mergeable_fields(other_item)
    MERGEABLE_FIELDS.each do |fieldname|
      new_value = merge_arrays(other_item, fieldname)
      delete_matching_query_row_numbers(new_value) if fieldname == :matching_queries
      send(putter_method(fieldname), new_value)
    end
{'fraction_non_alphanumeric': 0.10884808013355593, 'fraction_numerical': 0.00044518642181413465, 'mean_word_length': 4.605739238927012, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3281066', 'n_tokens_mistral': 3091, 'n_tokens_neox': 2986, 'n_words': 661}