<commit_before>require 'rubygems'
require 'mechanize'

#require 'JSON'

conference_id = '8265468'
easy_chair_url = "https://www.easychair.org/conferences/conference_change_yes.cgi?a=#{conference_id}"
easy_chair_review_url = "https://www.easychair.org/conferences/review_list.cgi?a=#{conference_id}"



u = ARGV[1] || ENV["RANDOM2015_username"]
pass = ARGV[2] || ENV["RANDOM2015_password"]


task :import_from_easychair => :environment do

  a = Mechanize.new { |agent|
    agent.user_agent_alias = 'Mac Safari'
  }
  a.pluggable_parser.default = Mechanize::Download

  a.get(easy_chair_url) do |page|

    # Submit the login form
    my_page = page.form_with(:action => 'verify.cgi') do |f|
      f['name']  = u
      f['password'] = <PASSWORD>
    end.submit

    review_page = a.get(easy_chair_review_url) do |review_page|
      form = review_page.form_with(:action => 'review_list.cgi')
      form.checkbox_with(:name => 'download').check
      response = form.submit
      papers = parse_papers(response.body)
      import_into_considerit(papers)
    end
  end
end

def import_into_considerit(papers)
  subdomain = Subdomain.find_by_name('RANDOM2015')
  user = User.find(210277) #find_by_email('<EMAIL>')

  for paper in papers

    is_new = false

    proposal = subdomain.proposals.find_by(slug: paper['url'])
    if !proposal
      proposal = Proposal.new(
        subdomain_id: subdomain.id,
        slug: paper['url'],
        user_id: user.id,
        published: true,
        cluster: 'Under Review'
      )
      is_new = true
    end

    fields = paper['description_fields']

    proposal.name = paper['topic']
    proposal.description = paper['description']
    proposal.description_fields = JSON.dump(fields)

    #pp paper
    if proposal.changed? || is_new
      if proposal.description_changed? || proposal.description_fields_changed?
        Notifier.create_notification 'edited', proposal, protagonist: user, subdomain: subdomain
        pp "#{proposal.slug} CHANGED!"
      end
      proposal.save
    end

  end


end

<commit_msg>need to save proposal before generating notification<commit_after>require 'rubygems'
require 'mechanize'

#require 'JSON'

conference_id = '8265468'
easy_chair_url = "https://www.easychair.org/conferences/conference_change_yes.cgi?a=#{conference_id}"
easy_chair_review_url = "https://www.easychair.org/conferences/review_list.cgi?a=#{conference_id}"



u = ARGV[1] || ENV["RANDOM2015_username"]
pass = ARGV[2] || ENV["<PASSWORD>"]


task :import_from_easychair => :environment do

  a = Mechanize.new { |agent|
    agent.user_agent_alias = 'Mac Safari'
  }
  a.pluggable_parser.default = Mechanize::Download

  a.get(easy_chair_url) do |page|

    # Submit the login form
    my_page = page.form_with(:action => 'verify.cgi') do |f|
      f['name']  = u
      f['password'] = <PASSWORD>
    end.submit

    review_page = a.get(easy_chair_review_url) do |review_page|
      form = review_page.form_with(:action => 'review_list.cgi')
      form.checkbox_with(:name => 'download').check
      response = form.submit
      papers = parse_papers(response.body)
      import_into_considerit(papers)
    end
  end
end

def import_into_considerit(papers)
  subdomain = Subdomain.find_by_name('RANDOM2015')
  user = User.find(311595) #210277) #find_by_email('<EMAIL>')

  for paper in papers

    is_new = false

    proposal = subdomain.proposals.find_by(slug: paper['url'])
    if !proposal
      proposal = Proposal.new(
        subdomain_id: subdomain.id,
        slug: paper['url'],
        user_id: user.id,
        published: true,
        cluster: 'Under Review'
      )
      is_new = true
    end

    fields = paper['description_fields']

    proposal.name = paper['topic']
    proposal.description = paper['description']
    proposal.description_fields = JSON.dump(fields)

    #pp paper
    if proposal.changed? || is_new
      text_updated = proposal.description_changed? || proposal.description_fields_changed?
      proposal.save
      
      if text_updated
        Notifier.create_notification 'edited', proposal, protagonist: user, subdomain: subdomain
        pp "#{proposal.slug} CHANGED!"
      end
    end

  end


end


{'fraction_non_alphanumeric': 0.11962750716332378, 'fraction_numerical': 0.013371537726838587, 'mean_word_length': 3.4850107066381155, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 16, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2245909', 'n_tokens_mistral': 1468, 'n_tokens_neox': 1389, 'n_words': 267}