<commit_before>require 'safe_yaml'
require 'open-uri'

require './lib/alerts'
require "./lib/git"

module FISC
  URL = "http://www.fisc.uscourts.gov/public-filings"

  def self.config
    @config ||= YAML.safe_load(File.read('config.yml'))
  end

  def self.url_for(page)
    # includes both docket activity and correspondence
    url = URL
    url << "?field_case_reference_nid=All"
    url << "&page=#{page}"
    url << "&t=#{Time.now.to_i}"
    url
  end


  def self.download!(page: 1)
    url = url_for page
    puts "Downloading: #{url}"
    open(
      url,
      "User-Agent" => "@FISACourt, twitter.com/FISACourt, github.com/konklone/fisacourt"
    ).read
  end

  def self.check!(options: {})
    return "test" if options[:test]

    if options[:use_file]
      body = File.read "./test/filings.html"
    else
      body = download!
    end

    # TODO: replace writing filing HTML to updating parsed data
    # filings_from_body! body
    File.open("docket/filings.html", "w") {|file| file.write body}

    puts "Saved current state of FISC docket."

    if FISC::Git.changed? or options[:test_error]
      begin
        raise Exception.new("Fake git error!") if test_error
        FISC::Git.save! "FISC dockets have been updated"
      rescue Exception => ex
        puts "Error doing the git commit and push! #{ex.inspect}"
        FISC::Alerts.admin! "Git error!"

        true
      end
    else
      false
    end

  end
end<commit_msg>Getting page number<commit_after>require 'open-uri'
require 'nokogiri'

require './lib/alerts'
require './lib/git'
require './lib/filings'

module FISC
  URL = "http://www.fisc.uscourts.gov/public-filings"

  def self.config
    @config ||= YAML.load(File.read('config.yml'))
  end

  # use the "last" link to figure out the final page #
  # pretty brittle: it'd better be there
  def self.last_page!
    puts "Finding page number of final page..."
    first = download! FISC::Filings.url_for(page: 1)
    doc = Nokogiri::HTML first
    link = doc.at("li.pager-last").at("a")['href']
    page = link.scan(/page=(\d+)/).first.first.to_i
    puts "Last page: #{page}"
    page
  end


  def self.download!(url)
    # puts "Downloading: #{url}"
    open(
      url,
      "User-Agent" => "@FISACourt, twitter.com/FISACourt, github.com/konklone/fisacourt"
    ).read
  end

  def self.check!(options: {})
    return "test" if options[:test]

    pages = options[:archive] ? (1..last_page!).to_a : [1]

    pages.each do |page|
      puts "[#{page}] Downloading filings..."
      if options[:use_file]
        body = File.read "./test/filings#{page}.html"
      else
        url = FISC::Filings.url_for page: page
        body = download! url
      end

      # parse filing data out of the HTML
      filings = FISC::Filings.for_page body

      puts filings

      # save a .yml file for each one into the docket dir
      # filings.each do |filing|
      #   FISC::Filings.save! filing
      # end
    end

    puts "Saved current state of FISC docket."

    if FISC::Git.changed? or options[:test_error]
      begin
        raise Exception.new("Fake git error!") if test_error
        FISC::Git.save! "FISC dockets have been updated"

      rescue Exception => ex
        puts "Error doing the git commit and push! #{ex.inspect}"
        FISC::Alerts.admin! "Git error!"
        true
      end
    else
      false
    end

  end
end
{'fraction_non_alphanumeric': 0.13057607090103399, 'fraction_numerical': 0.0011816838995568684, 'mean_word_length': 2.549266247379455, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5340718', 'n_tokens_mistral': 1186, 'n_tokens_neox': 1129, 'n_words': 311}