<commit_before>require 'cgi'
require 'date'

module Riki
  #
  # Represents a MediaWiki page. Only the latest revision of a page is considered.
  #
  class Page < Base
    class << self
      def find_by_title(titles)
        titles = Array(titles) # always treat titles as array
        return [] if titles.empty?

        results = {}

        # find cached pages
        titles.each{|title|
          cached = Riki::Base.cache.read(cache_key("page_#{title}"))
          if cached
            results[title] = cached
            Riki.logger.debug "Found cached version of requested page #{title} as '#{cached.title}'"
          end
        }

        # Check _in one coarse-grained API call which cached pages are still current
        if results.any?
          query = retrieve_pages(results.keys, 'timestamp')
          query.find('m:pages/m:page').each{|page|
            title = page['title'] # will already be the normalized and redirected title

            # Make sure redirects and normalizations are still current
            [normalization(query, title), redirection(query, title)].each do |indirection|
              next unless indirection
              if Riki::Base.cache.read(cache_key("page_#{indirection['from']}")).title != indirection['to']
                Riki.logger.debug "Redirection source #{indirection['from']} is stale. New target is #{indirection['to']}"
                results.delete(indirection['from'])
                titles.delete(indirection['from'])
                Riki::Base.cache.delete(cache_key("page_#{indirection['from']}"))

                # Re-check whether we have the new indirection target in our cache
                if cached_indirection_to = Riki::Base.cache.read(cache_key("page_#{indirection['to']}"))
                  results[title] = cached_indirection_to
                else
                  titles << indirection['to']
                end
              else
                Riki.logger.debug "Redirection is still current, no need to re-fetch #{indirection['from']}"
                titles.delete(indirection['from'])
              end
            end

            if results[title] # anything left to check after resolving indirections?
              last_modified = DateTime.strptime(page.find_first('m:revisions/m:rev')['timestamp'], '%Y-%m-%dT%H:%M:%S%Z')
              if last_modified < results[title].last_modified
                Riki.logger.debug "Cached version of page '#{title}' is from #{results[title].last_modified}, but an updated version is available that dates #{last_modified}"
                results.delete(title)
              else
                Riki.logger.debug "Cached version of page '#{title}' is still current. No need to re-fetch."
                titles.delete(title)
              end
            end
          }
        end

        Riki.logger.debug "Currency check leaves these pages to retrieve in full: #{titles.join(', ')}"

        return results.values if titles.empty? # no titles asked for or all results cached and current

        query = retrieve_pages(titles, ['content', 'timestamp'])
        query.find('m:pages/m:page').each{|page|
          if page['missing']
            Riki.logger.error "Page '#{page['title']}' was not found"
            next
          end

          validate!(page)

          p = Page.new(page['title'])

          p.id = page['pageid'].to_i
          p.namespace = page['ns']

          rev = page.find_first('m:revisions/m:rev')
          p.content = rev.content
          p.last_modified = DateTime.strptime(rev['timestamp'], '%Y-%m-%dT%H:%M:%S%Z')

          Riki::Base.cache.write(cache_key("page_#{p.title}"), p)
          Riki.logger.debug "Page '#{p.title}' written to the cache"

          # Cache the non-normalized form so that the next query will hit the cache even if asking for the non-normalized version
          # <normalized><n from="ISO_639-2" to="ISO 639-2" /></normalized>
          normalization = normalization(query, p.title)

          if normalization
            Riki.logger.debug "Also caching page '#{p.title}' under its non-normalized title '#{normalization['from']}'"
            p.normalized_from = normalization['from']

            # TODO We might save some space by caching a flyweight of p here (like an alias)
            Riki::Base.cache.write(cache_key("page_#{p.normalized_from}"), p)
          end

          # Cache the page under the title of the redirect source
          # <redirects><r from="&quot;Mimia&quot;" to="Mimipiscis" tofragment=""/>
          redirection = redirection(query, p.title)

          if redirection
            Riki.logger.debug "Also caching page '#{p.title}' under its redirect source '#{redirection['from']}'"
            p.redirected_from = redirection['from']

            # TODO We might save some space by caching a flyweight of p here (like an alias)
            Riki::Base.cache.write(cache_key("page_#{p.redirected_from}"), p)
          end

          results[p.title] = p
        }
<commit_msg>Simplify indirection checking<commit_after>require 'cgi'
require 'date'

module Riki
  #
  # Represents a MediaWiki page. Only the latest revision of a page is considered.
  #
  class Page < Base
    class << self
      def find_by_title(titles)
        titles = Array(titles) # always treat titles as array
        return [] if titles.empty?

        results = {}

        # find cached pages
        titles.each{|title|
          cached = Riki::Base.cache.read(cache_key("page_#{title}"))
          if cached
            results[title] = cached
            Riki.logger.debug "Found cached version of requested page #{title} as '#{cached.title}'"
          end
        }

        # Check _in one coarse-grained API call which cached pages are still current
        if results.any?
          query = retrieve_pages(results.keys, 'timestamp')
          query.find('m:pages/m:page').each{|page|
            title = page['title'] # will already be the normalized and redirected title

            # Make sure redirects and normalizations are still current
            [redirection(query, title), normalization(query, title)].each do |indirection|
              next unless indirection

              if Riki::Base.cache.read(cache_key("page_#{indirection['from']}")).title != indirection['to']
                Riki.logger.debug "Redirection source #{indirection['from']} is stale. New target is #{indirection['to']}"
                results.delete(indirection['from'])
                titles.delete(indirection['from'])
                Riki::Base.cache.delete(cache_key("page_#{indirection['from']}"))

                # Re-check whether we have the new indirection target in our cache
                if cached_indirection_to = Riki::Base.cache.read(cache_key("page_#{indirection['to']}"))
                  results[title] = cached_indirection_to
                else
                  titles << indirection['to']
                end
              else
                Riki.logger.debug "Redirection is still current, no need to re-fetch #{indirection['from']}"
                titles.delete(indirection['from'])
              end
            end

            if results[title] # anything left to check after resolving indirections?
              last_modified = DateTime.strptime(page.find_first('m:revisions/m:rev')['timestamp'], '%Y-%m-%dT%H:%M:%S%Z')
              if last_modified < results[title].last_modified
                Riki.logger.debug "Cached version of page '#{title}' is from #{results[title].last_modified}, but an updated version is available that dates #{last_modified}"
                results.delete(title)
              else
                Riki.logger.debug "Cached version of page '#{title}' is still current. No need to re-fetch."
                titles.delete(title)
              end
            end
          }
        end

        Riki.logger.debug "Currency check leaves these pages to retrieve in full: #{titles.join(', ')}"

        return results.values if titles.empty? # no titles asked for or all results cached and current

        query = retrieve_pages(titles, ['content', 'timestamp'])
        query.find('m:pages/m:page').each{|page|
          if page['missing']
            Riki.logger.error "Page '#{page['title']}' was not found"
            next
          end

          validate!(page)

          p = Page.new(page['title'])

          p.id = page['pageid'].to_i
          p.namespace = page['ns']

          rev = page.find_first('m:revisions/m:rev')
          p.content = rev.content
          p.last_modified = DateTime.strptime(rev['timestamp'], '%Y-%m-%dT%H:%M:%S%Z')

          Riki::Base.cache.write(cache_key("page_#{p.title}"), p)
          Riki.logger.debug "Page '#{p.title}' written to the cache"

          # Cache the non-normalized form so that the next query will hit the cache even if asking for the non-normalized version
          # <normalized><n from="ISO_639-2" to="ISO 639-2" /></normalized>
          if normalization = normalization(query, p.title)
            Riki.logger.debug "Also caching page '#{p.title}' under its non-normalized title '#{normalization['from']}'"
            p.normalized_from = normalization['from']

            # TODO We might save some space by caching a flyweight of p here (like an alias)
            Riki::Base.cache.write(cache_key("page_#{p.normalized_from}"), p)
          end

          # Cache the page under the title of the redirect source
          # <redirects><r from="&quot;Mimia&quot;" to="Mimipiscis" tofragment=""/>
          if redirection = redirection(query, p.title)
            Riki.logger.debug "Also caching page '#{p.title}' under its redirection source '#{redirection['from']}'"
            p.redirected_from = redirection['from']

            # TODO We might save some space by caching a flyweight of p here (like an alias)
            Riki::Base.cache.write(cache_key("page_#{p.redirected_from}"), p)
          end

          results[p.title] = p
        }

{'fraction_non_alphanumeric': 0.11787072243346007, 'fraction_numerical': 0.0016009605763458075, 'mean_word_length': 2.2409208819714657, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '716537', 'n_tokens_mistral': 2796, 'n_tokens_neox': 2699, 'n_words': 762}