<commit_before>require 'optparse'
require 'set'
require 'open-uri'
require 'fileutils'

module Hdo
  module Import
    class CLI
      attr_reader :options

      def initialize(argv)
        if argv.empty?
          raise ArgumentError, 'no args'
        else
          @options = parse_options argv
          @cmd     = argv.shift
          @rest    = argv
        end
      end

      def run
        case @cmd
        when 'json'
          import_files
        when 'daily'
          import_daily
        when 'api'
          import_api
        when 'dev'
          import_api(30)
        when 'representatives'
          import_api_representatives
        when 'votes'
          import_api_votes
        when 'promises'
          import_promises
        when 'parliament_issues'
          import_parliament_issues
        else
          raise ArgumentError, "unknown command: #{@cmd.inspect}"
        end
      end

      private

      def import_parliament_issues
        parliament_issues = parsing_data_source.parliament_issues(@options[:session])
        persister.import_parliament_issues parliament_issues
      end

      def import_api(vote_limit = nil)
        persister.import_parties parsing_data_source.parties(@options[:session])
        persister.import_committees parsing_data_source.committees(@options[:session])
        persister.import_districts parsing_data_source.districts
        persister.import_categories parsing_data_source.categories
        persister.import_parliament_periods parsing_data_source.parliament_periods
        persister.import_parliament_sessions parsing_data_source.parliament_sessions

        import_api_representatives
        import_api_votes(vote_limit)
      end

      def import_daily
        persister.import_parties parsing_data_source.parties(@options[:session])
        persister.import_committees parsing_data_source.committees(@options[:session])
        persister.import_districts parsing_data_source.districts
        persister.import_categories parsing_data_source.categories
        persister.import_parliament_periods parsing_data_source.parliament_periods
        persister.import_parliament_sessions parsing_data_source.parliament_sessions

        import_api_representatives

        parliament_issues = parsing_data_source.parliament_issues(@options[:session])
        persister.import_parliament_issues parliament_issues

        each_vote_for(parsing_data_source, parliament_issues) do |votes|
          persister.import_votes votes, infer: false
        end

        persister.infer_all_votes
        purge_cache
      end

      def import_api_votes(vote_limit = nil)
        parliament_issues = parsing_data_source.parliament_issues(@options[:session])

        if @options[:parliament_issue_ids]
          parliament_issues = parliament_issues.select { |i| @options[:parliament_issue_ids].include? i.external_id }
        end

        persister.import_parliament_issues parliament_issues

        each_vote_for(parsing_data_source, parliament_issues, vote_limit) do |votes|
          persister.import_votes votes, infer: false
        end

        persister.infer_all_votes
      end

      def import_api_representatives
        representatives = {}

        # the information in 'representatives_today' is more complete,
        # so it takes precedence

        parsing_data_source.representatives_today.each do |rep|
          representatives[rep.external_id] = rep
        end

        parsing_data_source.representatives(@options[:period]).each do |rep|
          representatives[rep.external_id] ||= rep
        end

        persister.import_representatives representatives.values
      end

      def import_promises
        spreadsheet = @rest.first or raise "no spreadsheet path given"

        promises = Hdo::StortingImporter::Promise.from_xlsx(spreadsheet)
        persister.import_promises promises
      end

      def each_vote_for(data_source, parliament_issues, limit = nil)
        count = 0

        parliament_issues.each_with_index do |parliament_issue, index|
          votes = data_source.votes_for(parliament_issue.external_id)
          count += votes.size

          yield votes
          break if limit && count >= limit

          parliament_issue_count = index + 1
          remaining_parliament_issues = parliament_issues.size - parliament_issue_count
          remaining_votes = (count / parliament_issue_count.to_f) * remaining_parliament_issues

          @log.info "->        #{count} votes for #{parliament_issue_count} parliament issues imported"
          @log.info "->        about #{remaining_votes.to_i} votes remaining for #{remaining_parliament_issues} parliament issues"
        end<commit_msg>Add code to fetch representative emails from https://scraperwiki.com/scrapers/hdo-representative-emails/<commit_after>require 'optparse'
require 'set'
require 'open-uri'
require 'fileutils'

module Hdo
  module Import
    class CLI
      attr_reader :options

      def initialize(argv)
        if argv.empty?
          raise ArgumentError, 'no args'
        else
          @options = parse_options argv
          @cmd     = argv.shift
          @rest    = argv
        end
      end

      def run
        case @cmd
        when 'json'
          import_files
        when 'daily'
          import_daily
        when 'api'
          import_api
        when 'dev'
          import_api(30)
        when 'representatives'
          import_api_representatives
        when 'votes'
          import_api_votes
        when 'promises'
          import_promises
        when 'parliament_issues', 'parliament-issues'
          import_parliament_issues
        when 'representative-emails'
          import_representative_emails
        else
          raise ArgumentError, "unknown command: #{@cmd.inspect}"
        end
      end

      private

      def import_parliament_issues
        parliament_issues = parsing_data_source.parliament_issues(@options[:session])
        persister.import_parliament_issues parliament_issues
      end

      def import_api(vote_limit = nil)
        persister.import_parties parsing_data_source.parties(@options[:session])
        persister.import_committees parsing_data_source.committees(@options[:session])
        persister.import_districts parsing_data_source.districts
        persister.import_categories parsing_data_source.categories
        persister.import_parliament_periods parsing_data_source.parliament_periods
        persister.import_parliament_sessions parsing_data_source.parliament_sessions

        import_api_representatives
        import_api_votes(vote_limit)
      end

      def import_daily
        persister.import_parties parsing_data_source.parties(@options[:session])
        persister.import_committees parsing_data_source.committees(@options[:session])
        persister.import_districts parsing_data_source.districts
        persister.import_categories parsing_data_source.categories
        persister.import_parliament_periods parsing_data_source.parliament_periods
        persister.import_parliament_sessions parsing_data_source.parliament_sessions

        import_api_representatives

        parliament_issues = parsing_data_source.parliament_issues(@options[:session])
        persister.import_parliament_issues parliament_issues

        each_vote_for(parsing_data_source, parliament_issues) do |votes|
          persister.import_votes votes, infer: false
        end

        persister.infer_all_votes
        purge_cache
      end

      def import_api_votes(vote_limit = nil)
        parliament_issues = parsing_data_source.parliament_issues(@options[:session])

        if @options[:parliament_issue_ids]
          parliament_issues = parliament_issues.select { |i| @options[:parliament_issue_ids].include? i.external_id }
        end

        persister.import_parliament_issues parliament_issues

        each_vote_for(parsing_data_source, parliament_issues, vote_limit) do |votes|
          persister.import_votes votes, infer: false
        end

        persister.infer_all_votes
      end

      def import_api_representatives
        representatives = {}

        # the information in 'representatives_today' is more complete,
        # so it takes precedence

        parsing_data_source.representatives_today.each do |rep|
          representatives[rep.external_id] = rep
        end

        parsing_data_source.representatives(@options[:period]).each do |rep|
          representatives[rep.external_id] ||= rep
        end

        persister.import_representatives representatives.values
      end

      def import_promises
        spreadsheet = @rest.first or raise "no spreadsheet path given"

        promises = Hdo::StortingImporter::Promise.from_xlsx(spreadsheet)
        persister.import_promises promises
      end

      SCRAPED_EMAIL_URL = "https://api.scraperwiki.com/api/1.0/datastore/sqlite?format=json&name=hdo-representative-emails&query=#{URI.escape 'select * from swdata'}"

      def import_representative_emails
        data = JSON.parse(RestClient.get(SCRAPED_EMAIL_URL))

        missing_emails = []
        data.each do |e|
          rep = Representative.where('first_name like ? and last_name = ?', "#{e['first_name'].strip}%", e['last_name'].strip).first

          if rep
            rep.email ||= e['email']
            rep.save!
          else
            missing_emails << e
          end
        end

        missing_reps = Representative.where(:email => nil).map(&:full_name)

        log.warn "representatives missing emails: #{missing_reps.to_json} "
        log.warn "emails missing representatives: #{missing_emails.to_json}"
      end

      def each_vote_for(data_source, parliament_issues, limit = nil)
        count = 0

        parliament_issues.each_with_index do |parliament_issue, index|
          votes = data_source.votes_for(parliament_issue.external_id)
          count += votes.size

          yield votes
          break if limit && count >= limit

          parliament_issue_count = index + 1
          remaining_parliament_issues = parliament_issues.size - parliament_issue_count
          remaining_votes = (count / parliament_issue_count.to_f) * remaining_parliament_issues

          log.info "->        #{count} votes for #{parliament_issue_count} parliament issues imported"
          log.info "->        about #{remaining_votes.to_i} votes remaining for #{remaining_parliament_issues} parliament issues"
        end
{'fraction_non_alphanumeric': 0.0652526989586319, 'fraction_numerical': 0.0009553835865099838, 'mean_word_length': 2.764113628191298, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1032521', 'n_tokens_mistral': 2969, 'n_tokens_neox': 2831, 'n_words': 550}