<commit_before>module Reality
  module Describers
    class Wikidata
      # Cache able to do `cache['Q212'] # => 'Ukraine'` and batch-fetching of new values.
      #
      # From external point of view, it just "knows" where to take what need labels for entity, fetch
      # all of it in as few requests as possible, and store in cache for further access as cache[wikidata_id]
      #
      # From internal point of view, it stores several different cache files separately:
      # * unit names;
      # * property names;
      # * other links from entity.
      #
      # The idea is to be able to cleanup / manage / update cache files separately ("other links"
      # can grew pretty large and probably needs rotation, for example).
      #
      class LabelsCache
        CACHE_PATH = File.expand_path('~/.reality/cache/wikidata')
        FACETS = %i[properties units entities]
        LINKS_PATH = [:*, 'mainsnak', 'datavalue', 'value', 'id']
        UNITS_PATH = [:*, 'mainsnak', 'datavalue', 'value', 'unit']

        def initialize(api)
          @api = api
          load_cache
        end

        def update_from(entity)
          claims = entity.fetch('claims')
          values = claims.values.flatten(1)
          cache_facets(
            properties: claims.keys,
            entities: Util.dig(values, *LINKS_PATH).compact,
            links: Util.dig(values, *UNITS_PATH).compact.grep_v('1').map(&method(:url2id))
          )<commit_msg>Bugfix<commit_after>module Reality
  module Describers
    class Wikidata
      # Cache able to do `cache['Q212'] # => 'Ukraine'` and batch-fetching of new values.
      #
      # From external point of view, it just "knows" where in entity structure labels are, fetches
      # all of them in as few requests as possible, and store in cache for further access as
      # `cache[wikidata_id]`.
      #
      # From internal point of view, it stores several different cache files separately:
      # * unit names;
      # * property names;
      # * other links from entity.
      #
      # The idea is to be able to cleanup / manage / update cache files separately ("other links"
      # can grew pretty large and probably needs rotation, for example).
      #
      class LabelsCache
        CACHE_PATH = File.expand_path('~/.reality/cache/wikidata')
        FACETS = %i[properties units entities]
        LINKS_PATH = [:*, 'mainsnak', 'datavalue', 'value', 'id']
        UNITS_PATH = [:*, 'mainsnak', 'datavalue', 'value', 'unit']

        def initialize(api)
          @api = api
          load_cache
        end

        def update_from(entity)
          claims = entity.fetch('claims')
          values = claims.values.flatten(1)
          cache_facets(
            properties: claims.keys,
            entities: Util.dig(values, *LINKS_PATH).grep_v('id').compact,
            units: Util.dig(values, *UNITS_PATH).compact.grep_v('1').map(&method(:url2id))
          )
{'fraction_non_alphanumeric': 0.10553278688524591, 'fraction_numerical': 0.004098360655737705, 'mean_word_length': 2.3744239631336406, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7312809', 'n_tokens_mistral': 858, 'n_tokens_neox': 833, 'n_words': 270}