<commit_before>class OaiController < ApplicationController
  Record = Struct.new(:id, :date, :pbcore)

  def index
    @records =
      RSolr.connect(url: 'http://localhost:8983/solr/')
      .get('select', params: {
             'q' => '*:*',
             'fl' => 'id,timestamp,xml',
             'rows' => '100' # Max for sitemaps
           })['response']['docs'].map do |d|
        Record.new(
          d['id'],
          d['timestamp'],
          d['xml'].gsub('<?xml version="1.0" encoding="UTF-8"?>', '').strip)
      end

    @response_date = '2002-06-01T19:20:30Z'
    @metadata_prefix = 'pbcore'

    respond_to do |format|<commit_msg>More validation / paging.<commit_after>class OaiController < ApplicationController
  Record = Struct.new(:id, :date, :pbcore)
  ROWS = 2

  def index
    @verb = params[:verb]
    fail("Unsupported verb: #{@verb}") unless @verb == 'ListRecords'

    @metadata_prefix = params[:metadataPrefix] || 'pbcore'
    fail('Unsupported metadataPrefix') unless @metadata_prefix == 'pbcore'

    resumption_token = params[:resumptionToken] || '0'
    fail("Unsupported resumptionToken: #{resumption_token}") unless resumption_token =~ /^\d*$/
    offset = resumption_token.to_i

    @response_date = Time.now.strftime('%FT%T')

    @records =
      RSolr.connect(url: 'http://localhost:8983/solr/')
      .get('select', params: {
             'q' => '*:*',
             'fl' => 'id,timestamp,xml',
             'rows' => ROWS,
             'offset' => offset
           })['response']['docs'].map do |d|
        Record.new(
          d['id'],
          d['timestamp'],
          d['xml'].gsub('<?xml version="1.0" encoding="UTF-8"?>', '').strip)
      end

    @next_resumption_token = # Not ideal: they'll need to go past the end.
      if @records.empty?
        nil
      else
        offset + ROWS
      end

    respond_to do |format|
{'fraction_non_alphanumeric': 0.17395944503735325, 'fraction_numerical': 0.017609391675560297, 'mean_word_length': 2.498134328358209, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 2, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7037458', 'n_tokens_mistral': 634, 'n_tokens_neox': 598, 'n_words': 124}