<commit_before>class MapReduce
  class TopicAuthority < MapReduce
    def all_set
      Channel.all
    end

    # TODO: convert to using topic ids
    # make sure to cache topics
    def map iterator
      iterator.ids.each do |ch_id|
        ch = Channel[ch_id]
        if ch.type == 'channel'
          Topic.ensure_for_channel(ch)
          Authority.all_from(ch).each do |authority|
            yield({topic: ch.slug_title, user_id: authority.user_id}, authority.to_f)
          end
          auth = (authority_from_added_facts(ch) + authority_from_followers(ch))
          if auth > 0
            yield({topic: ch.slug_title, user_id: ch.created_by_id}, auth)
          end
        end
      end
    end

    def authority_from_followers ch
      ch.containing_channels.count
    end

    def authority_from_added_facts ch
      [ch.sorted_internal_facts.count.to_f / 10, 5].min
    end

    def reduce bucket, values
      values.inject(0, :+)
    end

    def write_output ident, value
      topic = Topic.by_slug(ident[:topic])
      gu = GraphUser[ident[:user_id]] # TODO use dead graph_user
      Authority.from(topic, for: gu) << value

      Resque.enqueue Commands::Topics::UpdateUserAuthority,
         gu.id, topic.slug_title, value
    end
  end
end
<commit_msg>use dead graph_user here as well<commit_after>class MapReduce
  class TopicAuthority < MapReduce
    def all_set
      Channel.all
    end

    # TODO: convert to using topic ids
    # make sure to cache topics
    def map iterator
      iterator.ids.each do |ch_id|
        ch = Channel[ch_id]
        if ch.type == 'channel'
          Topic.ensure_for_channel(ch)
          Authority.all_from(ch).each do |authority|
            yield({topic: ch.slug_title, user_id: authority.user_id}, authority.to_f)
          end
          auth = (authority_from_added_facts(ch) + authority_from_followers(ch))
          if auth > 0
            yield({topic: ch.slug_title, user_id: ch.created_by_id}, auth)
          end
        end
      end
    end

    def authority_from_followers ch
      ch.containing_channels.count
    end

    def authority_from_added_facts ch
      [ch.sorted_internal_facts.count.to_f / 10, 5].min
    end

    def reduce bucket, values
      values.inject(0, :+)
    end

    def write_output ident, value
      topic = Topic.by_slug(ident[:topic])
      gu = DeadGraphUser.new(ident[:user_id])
      Authority.from(topic, for: gu) << value

      Resque.enqueue Commands::Topics::UpdateUserAuthority,
         gu.id, topic.slug_title, value
    end
  end
end

{'fraction_non_alphanumeric': 0.08444096950742767, 'fraction_numerical': 0.003909304143862392, 'mean_word_length': 2.358267716535433, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3512214', 'n_tokens_mistral': 834, 'n_tokens_neox': 813, 'n_words': 197}