<commit_before>#!/usr/bin/env ruby

require 'rubygems'
require 'yaml'
require 'fileutils'
require 'open-uri'

# install these gems:
require 'git'
require 'twitter'
require 'pony'
require 'twilio-rb'
require 'pushover'

# working directory should always be next to this script, to read in config.yml
FileUtils.chdir File.dirname(__FILE__)

def config
  @config ||= YAML.load(File.read("config.yml"))
end

if config['twitter']
  Twitter.configure do |twitter|
    twitter.consumer_key = config['twitter']['consumer_key']
    twitter.consumer_secret = config['twitter']['consumer_secret']
    twitter.oauth_token = config['twitter']['oauth_token']
    twitter.oauth_token_secret = config['twitter']['oauth_token_secret']
  end
end

if config['twilio']
  Twilio::Config.setup(
    account_sid: config['twilio']['account_sid'],
    auth_token: config['twilio']['auth_token']
  )
end

if config['pushover']
  Pushover.configure do |pushover|
    pushover.user = config['pushover']['user_key']
    pushover.token = config['pushover']['app_key']
  end
end

# ensure git repo/branch for docket is checked out and ready
# assumptions: remote exists, and branch exists on remote
@docket = "docket"
if !File.exists?(@docket)
  puts "[git] Cloning docket branch..."
  system "git clone --branch #{config['docket']['branch']} --single-branch #{config['docket']['remote']} #{@docket}"
end
puts "[git] Switching to docket branch..."
system "cd #{@docket} && git checkout #{config['docket']['branch']}"
puts "[git] Pulling latest changes..."
system "cd #{@docket} && git pull --no-edit"
@git = Git.open @docket


# check FISA court for updates, compare to last check
def check_fisa(test: false, test_error: false, use_file: false)
  return "test" if test

  if use_file
    body = File.read "./test-fisa.html"
  else
    puts "Downloading FISC docket..."
    body = open(
      "http://www.uscourts.gov/uscourts/courts/fisc/index.html?t=#{Time.now.to_i}",
      "User-Agent" => "@FISACourt, twitter.com/FISACourt, github.com/konklone/fisa"
    ).read
  end

  open("#{@docket}/fisa.html", "wt") do |file|
    file.write body
    file.close
  end

  puts "Saved current state of FISC docket."

  if changed? or test_error
    begin

      message = "FISC dockets have been updated"
      puts "Committing with message: #{message}"

      @git.add "fisa.html"

      response = @git.commit message
      sha = @git.gcommit(response.split(/[ \[\]]/)[2]).sha
      puts "[#{sha}] Committed update"

      system "cd #{@docket} && git push"
      puts "[#{sha}] Pushed changes."
<commit_msg>move URL to top, I guess<commit_after>#!/usr/bin/env ruby

require 'rubygems'
require 'yaml'
require 'fileutils'
require 'open-uri'

# install these gems:
require 'git'
require 'twitter'
require 'pony'
require 'twilio-rb'
require 'pushover'

# the URL we're tracking
FISA_URL = "http://www.fisc.uscourts.gov/public-filings"


# working directory should always be next to this script, to read in config.yml
FileUtils.chdir File.dirname(__FILE__)

def config
  @config ||= YAML.load(File.read("config.yml"))
end

if config['twitter']
  Twitter.configure do |twitter|
    twitter.consumer_key = config['twitter']['consumer_key']
    twitter.consumer_secret = config['twitter']['consumer_secret']
    twitter.oauth_token = config['twitter']['oauth_token']
    twitter.oauth_token_secret = config['twitter']['oauth_token_secret']
  end
end

if config['twilio']
  Twilio::Config.setup(
    account_sid: config['twilio']['account_sid'],
    auth_token: config['twilio']['auth_token']
  )
end

if config['pushover']
  Pushover.configure do |pushover|
    pushover.user = config['pushover']['user_key']
    pushover.token = config['pushover']['app_key']
  end
end

# ensure git repo/branch for docket is checked out and ready
# assumptions: remote exists, and branch exists on remote
@docket = "docket"
if !File.exists?(@docket)
  puts "[git] Cloning docket branch..."
  system "git clone --branch #{config['docket']['branch']} --single-branch #{config['docket']['remote']} #{@docket}"
end
puts "[git] Switching to docket branch..."
system "cd #{@docket} && git checkout #{config['docket']['branch']}"
puts "[git] Pulling latest changes..."
system "cd #{@docket} && git pull --no-edit"
@git = Git.open @docket


# check FISA court for updates, compare to last check
def check_fisa(test: false, test_error: false, use_file: false)
  return "test" if test

  if use_file
    body = File.read "./test-fisa.html"
  else
    puts "Downloading FISC docket..."
    body = open(
      "#{FISA_URL}?t=#{Time.now.to_i}",
      "User-Agent" => "@FISACourt, twitter.com/FISACourt, github.com/konklone/fisacourt"
    ).read
  end

  open("#{@docket}/fisa.html", "wt") do |file|
    file.write body
    file.close
  end

  puts "Saved current state of FISC docket."

  if changed? or test_error
    begin

      message = "FISC dockets have been updated"
      puts "Committing with message: #{message}"

      @git.add "fisa.html"

      response = @git.commit message
      sha = @git.gcommit(response.split(/[ \[\]]/)[2]).sha
      puts "[#{sha}] Committed update"

      system "cd #{@docket} && git push"
      puts "[#{sha}] Pushed changes."

{'fraction_non_alphanumeric': 0.15460146322680016, 'fraction_numerical': 0.0003850596842510589, 'mean_word_length': 4.043689320388349, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2504979', 'n_tokens_mistral': 1766, 'n_tokens_neox': 1702, 'n_words': 471}