<commit_before>require "flickr/base_converter"

module Flickr
  class Object

    class Photo

      attributes.add_locations(
        uploaded_at: [
          -> { @attributes["dateuploaded"] },
          -> { @attributes["dateupload"] },
        ],
        favorite: [
          -> { @attributes["isfavorite"] },
        ],
        posted_at: [
          -> { @attributes["dates"]["posted"] },
        ],
        taken_at: [
          -> { @attributes["dates"]["taken"] },
          -> { @attributes["datetaken"] }
        ],
        taken_at_granularity: [
          -> { @attributes["dates"]["takengranularity"] },
          -> { @attributes["datetakengranularity"] },
        ],
        updated_at: [
          -> { @attributes["dates"]["lastupdate"] },
          -> { @attributes["lastupdate"] },
        ],
        views_count: [
          -> { @attributes["views"] },
        ],
        public_editability: [
          -> { @attributes["publiceditability"] },
        ],
        comments_count: [
          -> { @attributes["comments"]["_content"] },
        ],
        has_people: [
          -> { @attributes["people"]["haspeople"] },
        ],
        safe: [
          -> { safety_level <= 1 },
        ],
        moderate: [
          -> { safety_level == 2 },
        ],
        restricted: [
          -> { safety_level == 3 },
        ],
        notes: [
          -> { @attributes["notes"]["note"] },
        ],
        tags: [
          -> {
            @attributes["tags"]["tag"].map do |hash|
              hash.merge("photo_id" => id)
            end
          },
          -> {
            tags = @attributes["tags"].split(" ").map do |content|
              {"_content" => content, "machine_tag" => 0}
            end
            machine_tags = @attributes["machine_tags"].split(" ").map do |content|
              {"_content" => content, "machine_tag" => 1}
            end

            tags + machine_tags
          },
        ],
        visibility: [
          -> {
            {
              "ispublic" => @attributes.fetch("ispublic"),
              "isfriend" => @attributes.fetch("isfriend"),
              "isfamily" => @attributes.fetch("isfamily"),
            }
          }
        ],
        title: [
          -> { @attributes["title"]["_content"] },
        ],
        description: [
          -> { @attributes["description"]["_content"] },
        ],
        owner: [
          -> {
            if @attributes["owner"].is_a?(String)
              {
                "nsid"       => @attributes["owner"],
                "username"   => @attributes["ownername"] || @attributes["username"],
                "iconserver" => @attributes["iconserver"],
                "iconfarm"   => @attributes["iconfarm"],
              }
            end
          },
        ],
        path_alias: [
          -> { @attributes["pathalias"] },
        ],
        location_visibility: [
          -> { @attributes["geoperms"] },
          -> {
            {
              "isfamily"  => @attributes.fetch("geo_is_family"),
              "isfriend"  => @attributes.fetch("geo_is_friend"),
              "iscontact" => @attributes.fetch("geo_is_contact"),
              "ispublic"  => @attributes.fetch("geo_is_public"),
            }
          }
        ],
        location: [
          -> {
            {
              "latitude"  => @attributes.fetch("latitude"),
              "longitude" => @attributes.fetch("longitude"),
              "accuracy"  => @attributes["accuracy"],
              "context"   => @attributes["context"],
              "place_id"  => @attributes["place_id"],
              "woeid"     => @attributes["woeid"],
            }
          },
        ],
        available_sizes: [
          -> {
            sizes = Size.all.select do |size|
              @attributes["url_#{size.abbreviation}"] or
              @attributes["sizes"] && @attributes["sizes"]["size"].find { |h| h["label"] == size.label }
            end

            SIZES.first(9) | sizes.map(&:name) | ["Original"]
          },
        ],
        largest_size: [<commit_msg>Only list sizes which are actually available<commit_after>require "flickr/base_converter"

module Flickr
  class Object

    class Photo

      attributes.add_locations(
        uploaded_at: [
          -> { @attributes["dateuploaded"] },
          -> { @attributes["dateupload"] },
        ],
        favorite: [
          -> { @attributes["isfavorite"] },
        ],
        posted_at: [
          -> { @attributes["dates"]["posted"] },
        ],
        taken_at: [
          -> { @attributes["dates"]["taken"] },
          -> { @attributes["datetaken"] }
        ],
        taken_at_granularity: [
          -> { @attributes["dates"]["takengranularity"] },
          -> { @attributes["datetakengranularity"] },
        ],
        updated_at: [
          -> { @attributes["dates"]["lastupdate"] },
          -> { @attributes["lastupdate"] },
        ],
        views_count: [
          -> { @attributes["views"] },
        ],
        public_editability: [
          -> { @attributes["publiceditability"] },
        ],
        comments_count: [
          -> { @attributes["comments"]["_content"] },
        ],
        has_people: [
          -> { @attributes["people"]["haspeople"] },
        ],
        safe: [
          -> { safety_level <= 1 },
        ],
        moderate: [
          -> { safety_level == 2 },
        ],
        restricted: [
          -> { safety_level == 3 },
        ],
        notes: [
          -> { @attributes["notes"]["note"] },
        ],
        tags: [
          -> {
            @attributes["tags"]["tag"].map do |hash|
              hash.merge("photo_id" => id)
            end
          },
          -> {
            tags = @attributes["tags"].split(" ").map do |content|
              {"_content" => content, "machine_tag" => 0}
            end
            machine_tags = @attributes["machine_tags"].split(" ").map do |content|
              {"_content" => content, "machine_tag" => 1}
            end

            tags + machine_tags
          },
        ],
        visibility: [
          -> {
            {
              "ispublic" => @attributes.fetch("ispublic"),
              "isfriend" => @attributes.fetch("isfriend"),
              "isfamily" => @attributes.fetch("isfamily"),
            }
          }
        ],
        title: [
          -> { @attributes["title"]["_content"] },
        ],
        description: [
          -> { @attributes["description"]["_content"] },
        ],
        owner: [
          -> {
            if @attributes["owner"].is_a?(String)
              {
                "nsid"       => @attributes["owner"],
                "username"   => @attributes["ownername"] || @attributes["username"],
                "iconserver" => @attributes["iconserver"],
                "iconfarm"   => @attributes["iconfarm"],
              }
            end
          },
        ],
        path_alias: [
          -> { @attributes["pathalias"] },
        ],
        location_visibility: [
          -> { @attributes["geoperms"] },
          -> {
            {
              "isfamily"  => @attributes.fetch("geo_is_family"),
              "isfriend"  => @attributes.fetch("geo_is_friend"),
              "iscontact" => @attributes.fetch("geo_is_contact"),
              "ispublic"  => @attributes.fetch("geo_is_public"),
            }
          }
        ],
        location: [
          -> {
            {
              "latitude"  => @attributes.fetch("latitude"),
              "longitude" => @attributes.fetch("longitude"),
              "accuracy"  => @attributes["accuracy"],
              "context"   => @attributes["context"],
              "place_id"  => @attributes["place_id"],
              "woeid"     => @attributes["woeid"],
            }
          },
        ],
        available_sizes: [
          -> {
            sizes = Size.all.select do |size|
              @attributes["url_#{size.abbreviation}"] or
              @attributes["sizes"] && @attributes["sizes"]["size"].find { |h| h["label"] == size.label }
            end

            sizes.map(&:name) | ["Original"]
          },
        ],
        largest_size: [
{'fraction_non_alphanumeric': 0.17306282082620386, 'fraction_numerical': 0.001344414568565143, 'mean_word_length': 1.3968951376684242, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 105, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5003596', 'n_tokens_mistral': 2220, 'n_tokens_neox': 2217, 'n_words': 283}