<commit_before>class Page < Base
  attr_reader :id, :author, :title, :container_title, :description, :license, :image_url, :tags, :issued, :updated_at

  def initialize(attributes, options={})
    @id = attributes.fetch("@id")
    @author = attributes.fetch("author", []).map { |a| { "given" => a["givenName"],
                                                         "family" => a["familyName"],
                                                         "orcid" => a["@id"] } }
    @title = attributes.fetch("name", nil)
    @container_title = attributes.fetch("publisher", nil)
    @description = attributes.fetch("description", nil)
    @license = attributes.fetch("license", nil)
    @image_url = attributes.fetch("image", nil)
    @tags = attributes.fetch("keywords", "").split(", ")
    @issued = attributes.fetch("datePublished", nil)
    @updated_at = @issued
  end

  def self.get_query_url(options={})
    url
  end

  def self.parse_data(result, options={})
    return nil if result.blank? || result['errors']

    items = result.fetch("data", [])

    if options[:id]
      item = items.find { |i| i["url"] == "https://#{options[:id]}/" }
      return nil if item.nil?

      { data: parse_item(item) }
    else
      items = items.select { |i| i.values.join("\n").downcase.include?(options[:query]) } if options[:query]
      items = items.select { |i| Array(i["tags"]).include?(options[:tag]) } if options[:tag]

      meta = { total: items.length, tags: parse_meta(items) }

      offset = (options[:offset] || 0).to_i
      rows = (options[:rows] || 25).to_i
      items = items[offset...offset + rows]

      { data: parse_items(items), meta: meta }
    end
  end

  def self.parse_meta(items)
    items.reduce({}) do |sum, i|
      i.fetch("keywords", "").split(", ").each { |tag| sum[tag] = sum[tag].to_i + 1 }
      sum
    end.sort_by {|_key, value| -value}[0..14].to_h
  end

  def self.url
    "#{ENV["BLOG_URL"]}/posts.json"
  end
end<commit_msg>fix pages lookup by tag<commit_after>class Page < Base
  attr_reader :id, :author, :title, :container_title, :description, :license, :image_url, :tags, :issued, :updated_at

  def initialize(attributes, options={})
    @id = attributes.fetch("@id")
    @author = attributes.fetch("author", []).map { |a| { "given" => a["givenName"],
                                                         "family" => a["familyName"],
                                                         "orcid" => a["@id"] } }
    @title = attributes.fetch("name", nil)
    @container_title = attributes.fetch("publisher", nil)
    @description = attributes.fetch("description", nil)
    @license = attributes.fetch("license", nil)
    @image_url = attributes.fetch("image", nil)
    @tags = attributes.fetch("keywords", "").split(", ")
    @issued = attributes.fetch("datePublished", nil)
    @updated_at = @issued
  end

  def self.get_query_url(options={})
    url
  end

  def self.parse_data(result, options={})
    return nil if result.blank? || result['errors']

    items = result.fetch("data", [])

    if options[:id]
      item = items.find { |i| i["@id"] == "https://doi.org/#{options[:id]}" }
      return nil if item.nil?

      { data: parse_item(item) }
    else
      items = items.select { |i| i.values.join("\n").downcase.include?(options[:query]) } if options[:query]
      items = items.select { |i| i.fetch("keywords", "").split(", ").include?(options[:tag]) } if options[:tag]

      meta = { total: items.length, tags: parse_meta(items) }

      offset = (options[:offset] || 0).to_i
      rows = (options[:rows] || 25).to_i
      items = items[offset...offset + rows]

      { data: parse_items(items), meta: meta }
    end
  end

  def self.parse_meta(items)
    items.reduce({}) do |sum, i|
      i.fetch("keywords", "").split(", ").each { |tag| sum[tag] = sum[tag].to_i + 1 }
      sum
    end.sort_by {|_key, value| -value}[0..14].to_h
  end

  def self.url
    "#{ENV["BLOG_URL"]}/posts.json"
  end
end
{'fraction_non_alphanumeric': 0.18369401107196778, 'fraction_numerical': 0.0035228988424760945, 'mean_word_length': 2.7288930581613506, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1023455', 'n_tokens_mistral': 1298, 'n_tokens_neox': 1257, 'n_words': 279}