<commit_before>require 'project_vote_smart'

# @see http://sunlightlabs.github.io/congress/
namespace :congress do
  def with_api_key
    if FederalLegislator.api_key
      yield
    else
      abort "FederalLegislator.api_key is not set"
    end
  end

  namespace :api do
    namespace :download do
      desc 'Download legislators from Congress API'
      task legislators: :environment do
        with_api_key do
          # get all possible matching officials (us senate, house) in 2 requests
          # rather than per created legislator
          pvs_api = ProjectVoteSmart.new
          officials = pvs_api.officials_by_state_and_office('us', [5])
          officials += api.officials_by_state_and_office('us', [6])

          CongressProcessor::Legislators.new.run officials: officials
        end
      end
    end
  end

  private
  class CongressProcessor
    attr_accessor :page, :per_page, :url, :api_key

    def initialize
      @page = 1
      @per_page = 50
      @url = "#{target_class.base_api_url}/#{target_class.api_plural_type}"
      @api_key = target_class.api_key
    end

    # override in subclass if different
    def target_class
      FederalLegislator
    end

    def params
      { apikey: api_key, page: page, per_page: per_page }
    end

    def run(options = {})
      json = JSON.parse(RestClient.get(url, params: params))
      json['results'].each { |result| process(result, options) }

      unless json['page']['count'] < per_page
        self.page = page + 1
        run options
      end
    end

    def process(result, options = {})
      # check for existing person, skip if there is one, at least for now
      unless target_class.where(id: result[target_class.api_id_field]).count > 0
        target_class.create_from_apis result, options
      end
    end

    class Legislators < CongressProcessor
      def fields
        'bioguide_id,first_name,middle_name,last_name,state,votesmart_id,email,gender,name_suffix,photo_url,twitter_id,chamber,district,party,terms'
      end

      def params
        super.merge({ fields: fields })
      end
    end
  end
end<commit_msg>Add facebook and youtube id data collection<commit_after>require 'project_vote_smart'

# @see http://sunlightlabs.github.io/congress/
namespace :congress do
  def with_api_key
    if FederalLegislator.api_key
      yield
    else
      abort "FederalLegislator.api_key is not set"
    end
  end

  namespace :api do
    namespace :download do
      desc 'Download legislators from Congress API'
      task legislators: :environment do
        with_api_key do
          # get all possible matching officials (us senate, house) in 2 requests
          # rather than per created legislator
          pvs_api = ProjectVoteSmart.new
          officials = pvs_api.officials_by_state_and_office('us', [5])
          officials += api.officials_by_state_and_office('us', [6])

          CongressProcessor::Legislators.new.run officials: officials
        end
      end
    end
  end

  private
  class CongressProcessor
    attr_accessor :page, :per_page, :url, :api_key

    def initialize
      @page = 1
      @per_page = 50
      @url = "#{target_class.base_api_url}/#{target_class.api_plural_type}"
      @api_key = target_class.api_key
    end

    # override in subclass if different
    def target_class
      FederalLegislator
    end

    def params
      { apikey: api_key, page: page, per_page: per_page }
    end

    def run(options = {})
      json = JSON.parse(RestClient.get(url, params: params))
      json['results'].each { |result| process(result, options) }

      unless json['page']['count'] < per_page
        self.page = page + 1
        run options
      end
    end

    def process(result, options = {})
      # check for existing person, skip if there is one, at least for now
      unless target_class.where(id: result[target_class.api_id_field]).count > 0
        target_class.create_from_apis result, options
      end
    end

    class Legislators < CongressProcessor
      def fields
        'bioguide_id,first_name,middle_name,last_name,state,votesmart_id,email,gender,name_suffix,photo_url,twitter_id,facebook_id,youtube_id,chamber,district,party,terms'
      end

      def params
        super.merge({ fields: fields })
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.08044640781213672, 'fraction_numerical': 0.00372006510113927, 'mean_word_length': 2.7572052401746725, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1424489', 'n_tokens_mistral': 1338, 'n_tokens_neox': 1288, 'n_words': 339}