<commit_before>require 'uri'

namespace :metrics do
  desc "Create metrics output"

  task :deeper => :environment do
    puts "Number\tName\tpositions\tpoints\tinclusions\tInclusions per point\tInclusions per position"
    Proposal.where(:domain_short => 'WA State').each do |p|
      printf("%i\t%s\t%i\t%i\t%i\t%.2f\t%.2f\n",
        p.designator,p.short_name,p.positions.published.count,p.points.published.count,p.inclusions.count,
        p.inclusions.count.to_f / p.points.published.count,
        p.inclusions.count.to_f / p.positions.published.count)
    end
  end

  task :basic => :environment do

    WASHINGTON = false
    if WASHINGTON
      years = [2010,2011,2012]
      accnt_id = 1
    else
      years = [2012]
      accnt_id = 2
    end

    puts "Overall activities"
    puts "Year\tusers\tpositions\tinclusions\tpoints\tcomments"
    years.each do |year|
      positions = Position.published.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8')
      users = User.where(:account_id => accnt_id).where("YEAR(created_at)=#{year} OR YEAR(last_sign_in_at)=#{year}").where('MONTH(created_at)>8')
      points = Point.published.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8')      
      inclusions = Inclusion.where(:account_id => accnt_id).where("YEAR(inclusions.created_at)=#{year}").where('MONTH(inclusions.created_at)>8').joins(:position).where('positions.published = 1')
      comments = Commentable::Comment.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8')

      printf("%i\t%s\t%i\t%i\t%i\t%i\n",
          year, users.count, positions.count, inclusions.count, points.count, comments.count)

    end

    puts "Distinct users engaging in each activity"
    puts "Year\tusers\tpositions\tinclusions\tpoints\tcomments"
    years.each do |year|
      positions = Position.published.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8').group(:user_id)
      users = User.where(:account_id => accnt_id).where("YEAR(created_at)=#{year} OR YEAR(last_sign_in_at)=#{year}").where('MONTH(created_at)>8')
      points = Point.published.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8').group(:user_id)
      inclusions = Inclusion.where(:account_id => accnt_id).where("YEAR(inclusions.created_at)=#{year}").where('MONTH(inclusions.created_at)>8').joins(:position).where('positions.published = 1').group("inclusions.user_id")
      comments = Commentable::Comment.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8').group(:user_id)

      printf("%i\t%s\t%i\t%i\t%i\t%i\n",
          year, 
          users.count, 
          positions.count.keys.length, 
          inclusions.count.keys.length, 
          points.count.keys.length, 
          comments.count.keys.length)

    end

    puts "Advanced metrics"
    puts "Year\tInclusions per point\tinclusions per position\tcomments per point\tpositions per user"
    years.each do |year|
      positions = Position.published.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8')
      users = User.where(:account_id => accnt_id).where("YEAR(created_at)=#{year} OR YEAR(last_sign_in_at)=#{year}").where('MONTH(created_at)>8')
      points = Point.published.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8')      
      inclusions = Inclusion.where(:account_id => accnt_id).where("YEAR(inclusions.created_at)=#{year}").where('MONTH(inclusions.created_at)>8').joins(:position).where('positions.published = 1')
      comments = Commentable::Comment.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8')

      printf("%i\t%.2f\t%.2f\t%.2f\t%.2f\n",
          year, 
          inclusions.count.to_f / points.count, #inclusions per point
          inclusions.count.to_f / positions.count, #inclusions per position
          comments.count.to_f / points.count, #comments per point
          positions.count.to_f / users.count #positions per user
      )
    end
  end

  task :fill_position_holes => :environment do 
    Point.published.where(:position_id => nil).each do |pnt|
      pnt.position = Position.where(:user_id => pnt.user_id).where(:proposal_id => pnt.proposal_id).last
      pnt.save
    end
    Inclusion.where(:position_id => nil).each do |inc|
      inc.position = Position.published.where(:user_id => inc.user_id).where(:proposal_id => inc.proposal_id).last
      inc.save
    end
  end

  task :normative_inclusion_overall => :environment do
    WASHINGTON = true
    if WASHINGTON
      years = [2010,2011,2012]
      accnt_id = 1
    else
      years = [2012]
      accnt_id = 2
    end

    puts "Normative metrics"
    puts "Year\tIncluded pro & con\tIncluded point by opposition\tIncluded for your side\tIncluded for other side\tConviction"
    years.each do |year|
      positions = Position.published.where(:account_id => accnt_id).where("YEAR(positions.created_at)=#{year}").where('MONTH(positions.created_at)>8').joins(:user).where('users.roles_mask=0')

      pro_and_con = 0
      not_pro_and_con = 0
      neither = 0
      inc_yr_side = 0
      inc_other_side = 0

      THRESH = 1
      conviction = 0

      positions.each do |pos|
        if pos.inclusions.count >= THRESH
          inc_points = pos.inclusions.joins(:point)
          conviction += pos.stance.abs
          if inc_points.where('points.is_pro=1').count > 0 && inc_points.where('points.is_pro=0').count > 0
            pro_and_con += 1
          else  
            not_pro_and_con += 1
          end

          if pos.stance > 0
            inc_yr_side += inc_points.where('points.is_pro=1').count
            inc_other_side += inc_points.where('points.is_pro=0').count
          elsif pos.stance < 0
            inc_yr_side += inc_points.where('points.is_pro=0').count
            inc_other_side += inc_points.where('points.is_pro=1').count
          end
        else
          neither += 1 
        end
      end

      printf("%i\t%.2f\t%.2f\t%i\t%i\t%.2f\n",
          year, 
          pro_and_con+not_pro_and_con > 0 ? pro_and_con.to_f / (pro_and_con+not_pro_and_con) : -1.0, #inclusions per point
          pro_and_con+not_pro_and_con > 0 ? pro_and_con.to_f / (pro_and_con+not_pro_and_con) : -1.0, #inclusions per position
          inc_yr_side, #inclusions per point
          inc_other_side, #inclusions per position        
          pro_and_con+not_pro_and_con > 0 ? conviction / (pro_and_con+not_pro_and_con) : -1.0
      )
    end

  end

  task :normative_inclusion_by_proposal => :environment do
    years = [2010,2011,2012]
    puts "Normative metrics"
    puts "Year\tProposal\tIncluded pro & con\tIncluded point by opposition\tIncluded for your side\tIncluded for other side\tConviction"
    
    puts "Year\tProposal\tIncluded pro & con\tConviction"

    years.each do |year|

      Proposal.where(:account_id => 1).tagged_with(["#{year}", 'state'], :all => true).each do |prop|

        positions = prop.positions.published.joins(:user).where('users.roles_mask=0')
                
        pro_and_con = 0
        not_pro_and_con = 0
        neither = 0
        inc_yr_side = 0
        inc_other_side = 0

        THRESH = 1

        #if positions.count < 100
        #  next
        #end

        contentiousness = 0.0

        positions.each do |pos|
          if pos.inclusions.count >= THRESH
            contentiousness += pos.stance.abs

            inc_points = pos.inclusions.joins(:point)
            if inc_points.where('points.is_pro=1').count > 0 && inc_points.where('points.is_pro=0').count > 0
              pro_and_con += 1
            else  
              not_pro_and_con += 1
            end

            if pos.stance > 0
              inc_yr_side += inc_points.where('points.is_pro=1').count
              inc_other_side += inc_points.where('points.is_pro=0').count
            elsif pos.stance < 0
              inc_yr_side += inc_points.where('points.is_pro=0').count
              inc_other_side += inc_points.where('points.is_pro=1').count
            end
          else
            neither += 1 
          end
        end

        # printf("%i\t%i\t%.2f\t%.2f\t%i\t%i\t%.2f\n",
        #     year, 
        #     prop.designator,
        #     pro_and_con.to_f / (pro_and_con+not_pro_and_con), #inclusions per point
        #     pro_and_con.to_f / (pro_and_con+not_pro_and_con), #inclusions per position
        #     inc_yr_side, #inclusions per point
        #     inc_other_side, #inclusions per position  
        #     contentiousness / (pro_and_con+not_pro_and_con)        
        # )

        printf("%i\t%i\t%s\t%.2f\t%.2f\n",
            year, 
            prop.designator,
            prop.short_name,
            pro_and_con.to_f / (pro_and_con+not_pro_and_con), #inclusions per point
            contentiousness / (pro_and_con+not_pro_and_con)        
        )        
      end
    end

  end

  task :fact_checking_basic => :environment do 
    puts "Fact-checking metrics"

    requests = Assessable::Request.all

    results = {
      'unknown' => {:pro => 0, :con => 0 },
      'supporter' => {:pro => 0, :con => 0 },
      'opposer' => {:pro => 0, :con => 0 },
      'neutral' => {:pro => 0, :con => 0 }

    }

    requests.each do |rq|
      requester = rq.user
      
      begin
        pos = requester.positions.published.where(:proposal_id => rq.root_object.proposal_id).last
      rescue
        next
      end

      is_pro = rq.root_object.is_pro ? :pro : :con

      if !pos
        stance = 'unknown'
      elsif pos.stance_bucket > 3
        stance = 'supporter'
      elsif pos.stance_bucket == 3
        stance = 'neutral'
      else
        stance = 'opposer'
      end

      results[stance][is_pro] += 1

    end
    pp results

  end

  task :fact_checking_rates => :environment do 
    puts "Computing average commenting rate and inclusion rate"

    Assessable::Assessment.all().each do |ass|
      #pp "Created at: #{ass.created_at}, Updated at: #{ass.updated_at}"
      #pp "requested at: #{ass.requests.first.created_at}"
    end
    average_date = Assessable::Assessment.select('FROM_UNIXTIME(AVG(UNIX_TIMESTAMP(created_at) + (UNIX_TIMESTAMP(updated_at) - UNIX_TIMESTAMP(created_at))/2 )) as updated_at')
    average_date = average_date[0].updated_at

    points = Assessable::Assessment.select('distinct(assessable_id)').map {|a| a.assessable_id }.compact

    proposals = points.map {|p| Point.find(p).proposal_id}.compact.uniq

    if nonfactchecked=false
      inclusions_all = Inclusion.where("proposal_id in (?) and point_id not in (?)", proposals, points)

      all_points = Point.where("proposal_id in (?)", proposals)
      comments_all = Commentable::Comment.where("commentable_id in (?) and commentable_id not in (?)", all_points.map {|p| p.id}.compact, points)

      all_views = PointListing.where("point_id in (?) and point_id not in (?)", all_points.map {|p| p.id}.compact, points)

      views_before = all_views.where('created_at < "' + average_date.to_s + '"').count
      views_after = all_views.where('created_at >= "' + average_date.to_s + '"').count

      comments_before = comments_all.where('created_at < "' + average_date.to_s + '"').count
      comments_after = comments_all.where('created_at >= "' + average_date.to_s + '"').count

      inclusions_before = inclusions_all.where('created_at < "' + average_date.to_s + '"').count
      inclusions_after = inclusions_all.where('created_at >= "' + average_date.to_s + '"').count
    else
      inclusions_all = Inclusion.where("point_id in (?)", points)

      comments_all = Commentable::Comment.where("commentable_id in (?)", points)

      all_views = PointListing.where("point_id in (?)", points)

      views_before = all_views.where('created_at < "' + average_date.to_s + '"').count
      views_after = all_views.where('created_at >= "' + average_date.to_s + '"').count

      comments_before = comments_all.where('created_at < "' + average_date.to_s + '"').count
      comments_after = comments_all.where('created_at >= "' + average_date.to_s + '"').count

      inclusions_before = inclusions_all.where('created_at < "' + average_date.to_s + '"').count
      inclusions_after = inclusions_all.where('created_at >= "' + average_date.to_s + '"').count
    end


    pp "BEFORE"
    pp "Views: #{views_before}"
    pp "Comments: #{comments_before}"
    pp "Rate: #{comments_before.to_f/views_before}"

    pp "Inclusions: #{inclusions_before}"
    pp "Rate: #{inclusions_before.to_f/views_before}"

    pp "AFTER"
    pp "Views: #{views_after}"
    pp "Comments: #{comments_after}"
    pp "Rate: #{comments_after.to_f/views_after}"

    pp "Inclusions: #{inclusions_after}"
    pp "Rate: #{inclusions_after.to_f/views_after}"

  end

  task :fact_checking => :environment do 
    puts "Fact-checking metrics"


    impacts = []
    [0,1,2].each do |verdict|
      puts "For points with an overall " + verdict.to_s + " verdict"
      col = :overall_verdict
      #col = :max_verdict
      assessments = Assessable::Assessment.where(col => verdict)

      inclusions = { :before => 0, :after => 0 } 
      views = { :before => 0, :after => 0 }

      normalized_impact = 0
      cnt = 0

      assessments.each do |ass|
        pivot = ass.updated_at
        point = ass.root_object

        #next if ass.claims.count > 1
        #next if point.point_listings.where('created_at > "' + pivot.to_s + '"').count < 10


        population_before = point.proposal.inclusions.where('created_at < "' + pivot.to_s + '"').count.to_f / point.proposal.point_listings.where('created_at < "' + pivot.to_s + '"').count
        population_after = point.proposal.inclusions.where('created_at > "' + pivot.to_s + '"').count.to_f / point.proposal.point_listings.where('created_at > "' + pivot.to_s + '"').count
        population_after = population_after.infinite? ? 1.0 : population_after

        point_before = point.inclusions.where('created_at < "' + pivot.to_s + '"').count.to_f / point.point_listings.where('created_at < "' + pivot.to_s + '"').count
        point_after = point.inclusions.where('created_at > "' + pivot.to_s + '"').count.to_f / point.point_listings.where('created_at > "' + pivot.to_s + '"').count
        point_after = point_after.infinite? ? 1.0 : point_after

        next if point_after.nan?

        normalized_impact += (point_before > 0 ? point_after / point_before : 0) - (population_before > 0 ? population_after / population_before : 0)
        inclusions[:before] += point.inclusions.where('created_at < "' + pivot.to_s + '"').count
        inclusions[:after] += point.inclusions.where('created_at > "' + pivot.to_s + '"') .count
        views[:before] += point.point_listings.where('created_at < "' + pivot.to_s + '"').count
        views[:after] += point.point_listings.where('created_at > "' + pivot.to_s + '"').count
        cnt += 1

        impacts.push([point, normalized_impact, cnt, point.point_listings.where('created_at > "' + pivot.to_s + '"').count, verdict])

      end

      printf("%s:\tinclusions %i\tviews %i\t%.3f\n", 'Before', inclusions[:before], views[:before], inclusions[:before].to_f / views[:before] )
      printf("%s:\tinclusions %i\tviews %i\t%.3f\n", 'After', inclusions[:after], views[:after], inclusions[:after].to_f / views[:after] )

      printf("Impact:\t%.3f\n", (inclusions[:after].to_f / views[:after]) / (inclusions[:before].to_f / views[:before]) )
      printf("Normalized:\t%.3f\n", normalized_impact / cnt )
      pp cnt


    end

    impacts.sort! {|a,b| a[1] <=> b[1]}

    impacts.each do |i|
      printf("%i\t%i\t%i\t%.3f\t%i\n", i[4], i[0].point_listings.count, i[3], i[1], i[0].id)
    end
  end

  task :fact_checking_discussion => :environment do 
    puts "Fact-checking metrics"


    impacts = []
    proposal_point_map = {}
    Point.where('comment_count > 0').each do |pnt|
      comts = pnt.comments.map {|x| x.created_at}.compact
      if !proposal_point_map.has_key? pnt.proposal.id
        proposal_point_map[pnt.proposal_id] = [comts]
      else
        proposal_point_map[pnt.proposal_id].push comts
      end
    end
    #pp proposal_point_map

    [0,1,2].each do |verdict|
      puts "For points with an overall " + verdict.to_s + " verdict"
      col = :overall_verdict
      #col = :max_verdict
      assessments = Assessable::Assessment.where(col => verdict)

      comments = { :before => 0, :after => 0 } 
      views = { :before => 0, :after => 0 }

      normalized_impact = 0
      cnt = 0


      assessments.each do |ass|
        pivot = ass.updated_at
        point = ass.root_object

        #next if point.comments.count == 0
        #next if ass.claims.count > 1
        #next if point.point_listings.where('created_at > "' + pivot.to_s + '"').count < 10

        comments_before_pivot = comments_after_pivot = 0.0

        proposal_point_map[point.proposal_id].each do |pnt|
          pnt.each do |comment|
            if comment < pivot
              comments_before_pivot += 1
            else  
              comments_after_pivot += 1
            end
          end 
        end

        point_before = point.comments.where('created_at < "' + pivot.to_s + '"').count.to_f / point.point_listings.where('created_at < "' + pivot.to_s + '"').count
        point_after = point.comments.where('created_at > "' + pivot.to_s + '"').count.to_f / point.point_listings.where('created_at > "' + pivot.to_s + '"').count
        point_after = point_after.infinite? ? 1.0 : point_after

        population_before = comments_before_pivot / point.proposal.point_listings.where('created_at < "' + pivot.to_s + '"').count
        population_after = comments_after_pivot / point.proposal.point_listings.where('created_at > "' + pivot.to_s + '"').count
        population_after = population_after.infinite? ? 1.0 : population_after

        
        next if point_after.nan?

        normalized_impact += (point_before > 0 ? point_after / point_before : 0) - (population_before > 0 ? population_after / population_before : 0)
        comments[:before] += point.comments.where('created_at < "' + pivot.to_s + '"').count
        comments[:after] += point.comments.where('created_at > "' + pivot.to_s + '"') .count
        views[:before] += point.point_listings.where('created_at < "' + pivot.to_s + '"').count
        views[:after] += point.point_listings.where('created_at > "' + pivot.to_s + '"').count
        cnt += 1

        impacts.push([point, normalized_impact, cnt, point.point_listings.where('created_at > "' + pivot.to_s + '"').count, verdict])

      end

      printf("%s:\tcomments %i\tviews %i\t%.3f\n", 'Before', comments[:before], views[:before], comments[:before].to_f / views[:before] )
      printf("%s:\tcomments %i\tviews %i\t%.3f\n", 'After', comments[:after], views[:after], comments[:after].to_f / views[:after] )

      printf("Impact:\t%.3f\n", (comments[:after].to_f / views[:after]) / (comments[:before].to_f / views[:before]) )
      printf("Normalized:\t%.3f\n", normalized_impact / cnt )
      pp cnt


    end

    impacts.sort! {|a,b| a[1] <=> b[1]}

    impacts.each do |i|
      printf("%i\t%i\t%i\t%.3f\t%i\n", i[4], i[0].point_listings.count, i[3], i[1], i[0].id)
    end
  end

  task :export_fact_checking => :environment do 

    election_date = DateTime.new(2012,11,8)

    Assessable::Assessment.all().each do |ass|
      #pp "Created at: #{ass.created_at}, Updated at: #{ass.updated_at}"
      #pp "requested at: #{ass.requests.first.created_at}"
    end
    # average_date = Assessable::Assessment.select('FROM_UNIXTIME(AVG(UNIX_TIMESTAMP(created_at) + (UNIX_TIMESTAMP(updated_at) - UNIX_TIMESTAMP(created_at))/2 )) as updated_at')
    # average_date = average_date[0].updated_at

    points = Assessable::Assessment.select('distinct(assessable_id)').map {|a| a.assessable_id }.compact

    proposals = points.map {|p| Point.find(p).proposal_id}.compact.uniq
    all_points = Point.where("proposal_id in (?) AND created_at < (?)", proposals, election_date)

    nf_inclusions_all = Inclusion.where("proposal_id in (?) and point_id not in (?)  AND created_at < (?)", proposals, points, election_date)

    nf_all_points = Point.where("proposal_id in (?)", proposals)
    nf_comments_all = Commentable::Comment.where("commentable_id in (?) and commentable_id not in (?) AND created_at < (?)", all_points.map {|p| p.id}.compact, points, election_date)

    nf_all_views = PointListing.where("point_id in (?) and point_id not in (?) and created_at is not null  AND created_at < (?)", all_points.map {|p| p.id}.compact, points, election_date)

    fc_inclusions_all = Inclusion.where("point_id in (?) AND created_at < (?)", points, election_date)

    fc_comments_all = Commentable::Comment.where("commentable_id in (?) AND created_at < (?)", points, election_date)

    fc_all_views = PointListing.where("point_id in (?) and created_at is not null AND created_at < (?)", points, election_date)

    tmp = {}
    nf_all_views.each do |vw|
      if !tmp.has_key? [vw.point_id, vw.user_id]
        tmp[[vw.point_id,vw.user_id]] = vw
      end
    end
    nf_all_views = tmp.values()

    tmp = {}
    fc_all_views.each do |vw|
      if !tmp.has_key? [vw.point_id, vw.user_id]
        tmp[[vw.point_id,vw.user_id]] = vw
      end
    end
    fc_all_views = tmp.values()


    views_before = 0
    views_after = 0

    inclusions_before = 0
    inclusions_after = 0

    comments_after = 0
    comments_before = 0

    views_before0 = 0
    views_before1 = 0
    views_before2 = 0

    views_after0 = 0
    views_after1 = 0
    views_after2 = 0

    inclusions_before0 = 0
    inclusions_before1 = 0
    inclusions_before2 = 0

    inclusions_after0 = 0
    inclusions_after1 = 0
    inclusions_after2 = 0

    simulate_fact_check = true
    simulated_date = DateTime.new(2012,10,28)
    simulated_assessment = Assessable::Assessment.new(:updated_at => simulated_date, :overall_verdict => -1)

    require 'csv'
    CSV.open("data.csv", "w") do |csv|

      head = ['firstview_timestamp', 'point_id',  'was_point_factchecked', 'factchecked_before_view', 'fact_check_accuracy_verdict', 'fact_check_timestamp', 'user_commented', 'user_commented_before_factcheck', 'user_commented_after_factcheck', 'user_included_point']


      csv << head
      [ [fc_all_views, true], [nf_all_views, false] ].each do |views, was_checked|
        views.each do |view|
          row = []

          # Time stamp for the view
          row.push view.created_at

          # An identifier for the point that was viewed
          row.push view.point_id

          # An indicator for whether or not the point was ever fact checked (regardless of whether the particular view occurred before or after the fact check)
          row.push was_checked

          # An indicator for whether or not the point had been fact checked prior to the view
          assessment = Assessable::Assessment.find_by_assessable_id(view.point_id)
          if simulate_fact_check && !was_checked
            assessment = simulated_assessment
          end

          row.push was_checked || simulate_fact_check ? assessment.updated_at > view.created_at : nil<commit_msg>data metric modifications for brian<commit_after>require 'uri'

namespace :metrics do
  desc "Create metrics output"

  task :deeper => :environment do
    puts "Number\tName\tpositions\tpoints\tinclusions\tInclusions per point\tInclusions per position"
    Proposal.where(:domain_short => 'WA State').each do |p|
      printf("%i\t%s\t%i\t%i\t%i\t%.2f\t%.2f\n",
        p.designator,p.short_name,p.positions.published.count,p.points.published.count,p.inclusions.count,
        p.inclusions.count.to_f / p.points.published.count,
        p.inclusions.count.to_f / p.positions.published.count)
    end
  end

  task :basic => :environment do

    WASHINGTON = false
    if WASHINGTON
      years = [2010,2011,2012]
      accnt_id = 1
    else
      years = [2012]
      accnt_id = 2
    end

    puts "Overall activities"
    puts "Year\tusers\tpositions\tinclusions\tpoints\tcomments"
    years.each do |year|
      positions = Position.published.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8')
      users = User.where(:account_id => accnt_id).where("YEAR(created_at)=#{year} OR YEAR(last_sign_in_at)=#{year}").where('MONTH(created_at)>8')
      points = Point.published.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8')      
      inclusions = Inclusion.where(:account_id => accnt_id).where("YEAR(inclusions.created_at)=#{year}").where('MONTH(inclusions.created_at)>8').joins(:position).where('positions.published = 1')
      comments = Commentable::Comment.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8')

      printf("%i\t%s\t%i\t%i\t%i\t%i\n",
          year, users.count, positions.count, inclusions.count, points.count, comments.count)

    end

    puts "Distinct users engaging in each activity"
    puts "Year\tusers\tpositions\tinclusions\tpoints\tcomments"
    years.each do |year|
      positions = Position.published.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8').group(:user_id)
      users = User.where(:account_id => accnt_id).where("YEAR(created_at)=#{year} OR YEAR(last_sign_in_at)=#{year}").where('MONTH(created_at)>8')
      points = Point.published.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8').group(:user_id)
      inclusions = Inclusion.where(:account_id => accnt_id).where("YEAR(inclusions.created_at)=#{year}").where('MONTH(inclusions.created_at)>8').joins(:position).where('positions.published = 1').group("inclusions.user_id")
      comments = Commentable::Comment.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8').group(:user_id)

      printf("%i\t%s\t%i\t%i\t%i\t%i\n",
          year, 
          users.count, 
          positions.count.keys.length, 
          inclusions.count.keys.length, 
          points.count.keys.length, 
          comments.count.keys.length)

    end

    puts "Advanced metrics"
    puts "Year\tInclusions per point\tinclusions per position\tcomments per point\tpositions per user"
    years.each do |year|
      positions = Position.published.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8')
      users = User.where(:account_id => accnt_id).where("YEAR(created_at)=#{year} OR YEAR(last_sign_in_at)=#{year}").where('MONTH(created_at)>8')
      points = Point.published.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8')      
      inclusions = Inclusion.where(:account_id => accnt_id).where("YEAR(inclusions.created_at)=#{year}").where('MONTH(inclusions.created_at)>8').joins(:position).where('positions.published = 1')
      comments = Commentable::Comment.where(:account_id => accnt_id).where("YEAR(created_at)=#{year}").where('MONTH(created_at)>8')

      printf("%i\t%.2f\t%.2f\t%.2f\t%.2f\n",
          year, 
          inclusions.count.to_f / points.count, #inclusions per point
          inclusions.count.to_f / positions.count, #inclusions per position
          comments.count.to_f / points.count, #comments per point
          positions.count.to_f / users.count #positions per user
      )
    end
  end

  task :fill_position_holes => :environment do 
    Point.published.where(:position_id => nil).each do |pnt|
      pnt.position = Position.where(:user_id => pnt.user_id).where(:proposal_id => pnt.proposal_id).last
      pnt.save
    end
    Inclusion.where(:position_id => nil).each do |inc|
      inc.position = Position.published.where(:user_id => inc.user_id).where(:proposal_id => inc.proposal_id).last
      inc.save
    end
  end

  task :normative_inclusion_overall => :environment do
    WASHINGTON = true
    if WASHINGTON
      years = [2010,2011,2012]
      accnt_id = 1
    else
      years = [2012]
      accnt_id = 2
    end

    puts "Normative metrics"
    puts "Year\tIncluded pro & con\tIncluded point by opposition\tIncluded for your side\tIncluded for other side\tConviction"
    years.each do |year|
      positions = Position.published.where(:account_id => accnt_id).where("YEAR(positions.created_at)=#{year}").where('MONTH(positions.created_at)>8').joins(:user).where('users.roles_mask=0')

      pro_and_con = 0
      not_pro_and_con = 0
      neither = 0
      inc_yr_side = 0
      inc_other_side = 0

      THRESH = 1
      conviction = 0

      positions.each do |pos|
        if pos.inclusions.count >= THRESH
          inc_points = pos.inclusions.joins(:point)
          conviction += pos.stance.abs
          if inc_points.where('points.is_pro=1').count > 0 && inc_points.where('points.is_pro=0').count > 0
            pro_and_con += 1
          else  
            not_pro_and_con += 1
          end

          if pos.stance > 0
            inc_yr_side += inc_points.where('points.is_pro=1').count
            inc_other_side += inc_points.where('points.is_pro=0').count
          elsif pos.stance < 0
            inc_yr_side += inc_points.where('points.is_pro=0').count
            inc_other_side += inc_points.where('points.is_pro=1').count
          end
        else
          neither += 1 
        end
      end

      printf("%i\t%.2f\t%.2f\t%i\t%i\t%.2f\n",
          year, 
          pro_and_con+not_pro_and_con > 0 ? pro_and_con.to_f / (pro_and_con+not_pro_and_con) : -1.0, #inclusions per point
          pro_and_con+not_pro_and_con > 0 ? pro_and_con.to_f / (pro_and_con+not_pro_and_con) : -1.0, #inclusions per position
          inc_yr_side, #inclusions per point
          inc_other_side, #inclusions per position        
          pro_and_con+not_pro_and_con > 0 ? conviction / (pro_and_con+not_pro_and_con) : -1.0
      )
    end

  end

  task :normative_inclusion_by_proposal => :environment do
    years = [2010,2011,2012]
    puts "Normative metrics"
    puts "Year\tProposal\tIncluded pro & con\tIncluded point by opposition\tIncluded for your side\tIncluded for other side\tConviction"
    
    puts "Year\tProposal\tIncluded pro & con\tConviction"

    years.each do |year|

      Proposal.where(:account_id => 1).tagged_with(["#{year}", 'state'], :all => true).each do |prop|

        positions = prop.positions.published.joins(:user).where('users.roles_mask=0')
                
        pro_and_con = 0
        not_pro_and_con = 0
        neither = 0
        inc_yr_side = 0
        inc_other_side = 0

        THRESH = 1

        #if positions.count < 100
        #  next
        #end

        contentiousness = 0.0

        positions.each do |pos|
          if pos.inclusions.count >= THRESH
            contentiousness += pos.stance.abs

            inc_points = pos.inclusions.joins(:point)
            if inc_points.where('points.is_pro=1').count > 0 && inc_points.where('points.is_pro=0').count > 0
              pro_and_con += 1
            else  
              not_pro_and_con += 1
            end

            if pos.stance > 0
              inc_yr_side += inc_points.where('points.is_pro=1').count
              inc_other_side += inc_points.where('points.is_pro=0').count
            elsif pos.stance < 0
              inc_yr_side += inc_points.where('points.is_pro=0').count
              inc_other_side += inc_points.where('points.is_pro=1').count
            end
          else
            neither += 1 
          end
        end

        # printf("%i\t%i\t%.2f\t%.2f\t%i\t%i\t%.2f\n",
        #     year, 
        #     prop.designator,
        #     pro_and_con.to_f / (pro_and_con+not_pro_and_con), #inclusions per point
        #     pro_and_con.to_f / (pro_and_con+not_pro_and_con), #inclusions per position
        #     inc_yr_side, #inclusions per point
        #     inc_other_side, #inclusions per position  
        #     contentiousness / (pro_and_con+not_pro_and_con)        
        # )

        printf("%i\t%i\t%s\t%.2f\t%.2f\n",
            year, 
            prop.designator,
            prop.short_name,
            pro_and_con.to_f / (pro_and_con+not_pro_and_con), #inclusions per point
            contentiousness / (pro_and_con+not_pro_and_con)        
        )        
      end
    end

  end

  task :fact_checking_basic => :environment do 
    puts "Fact-checking metrics"

    requests = Assessable::Request.all

    results = {
      'unknown' => {:pro => 0, :con => 0 },
      'supporter' => {:pro => 0, :con => 0 },
      'opposer' => {:pro => 0, :con => 0 },
      'neutral' => {:pro => 0, :con => 0 }

    }

    requests.each do |rq|
      requester = rq.user
      
      begin
        pos = requester.positions.published.where(:proposal_id => rq.root_object.proposal_id).last
      rescue
        next
      end

      is_pro = rq.root_object.is_pro ? :pro : :con

      if !pos
        stance = 'unknown'
      elsif pos.stance_bucket > 3
        stance = 'supporter'
      elsif pos.stance_bucket == 3
        stance = 'neutral'
      else
        stance = 'opposer'
      end

      results[stance][is_pro] += 1

    end
    pp results

  end

  task :fact_checking_rates => :environment do 
    puts "Computing average commenting rate and inclusion rate"

    Assessable::Assessment.all().each do |ass|
      #pp "Created at: #{ass.created_at}, Updated at: #{ass.updated_at}"
      #pp "requested at: #{ass.requests.first.created_at}"
    end
    average_date = Assessable::Assessment.select('FROM_UNIXTIME(AVG(UNIX_TIMESTAMP(created_at) + (UNIX_TIMESTAMP(updated_at) - UNIX_TIMESTAMP(created_at))/2 )) as updated_at')
    average_date = average_date[0].updated_at

    points = Assessable::Assessment.select('distinct(assessable_id)').map {|a| a.assessable_id }.compact

    proposals = points.map {|p| Point.find(p).proposal_id}.compact.uniq

    if nonfactchecked=false
      inclusions_all = Inclusion.where("proposal_id in (?) and point_id not in (?)", proposals, points)

      all_points = Point.where("proposal_id in (?)", proposals)
      comments_all = Commentable::Comment.where("commentable_id in (?) and commentable_id not in (?)", all_points.map {|p| p.id}.compact, points)

      all_views = PointListing.where("point_id in (?) and point_id not in (?)", all_points.map {|p| p.id}.compact, points)

      views_before = all_views.where('created_at < "' + average_date.to_s + '"').count
      views_after = all_views.where('created_at >= "' + average_date.to_s + '"').count

      comments_before = comments_all.where('created_at < "' + average_date.to_s + '"').count
      comments_after = comments_all.where('created_at >= "' + average_date.to_s + '"').count

      inclusions_before = inclusions_all.where('created_at < "' + average_date.to_s + '"').count
      inclusions_after = inclusions_all.where('created_at >= "' + average_date.to_s + '"').count
    else
      inclusions_all = Inclusion.where("point_id in (?)", points)

      comments_all = Commentable::Comment.where("commentable_id in (?)", points)

      all_views = PointListing.where("point_id in (?)", points)

      views_before = all_views.where('created_at < "' + average_date.to_s + '"').count
      views_after = all_views.where('created_at >= "' + average_date.to_s + '"').count

      comments_before = comments_all.where('created_at < "' + average_date.to_s + '"').count
      comments_after = comments_all.where('created_at >= "' + average_date.to_s + '"').count

      inclusions_before = inclusions_all.where('created_at < "' + average_date.to_s + '"').count
      inclusions_after = inclusions_all.where('created_at >= "' + average_date.to_s + '"').count
    end


    pp "BEFORE"
    pp "Views: #{views_before}"
    pp "Comments: #{comments_before}"
    pp "Rate: #{comments_before.to_f/views_before}"

    pp "Inclusions: #{inclusions_before}"
    pp "Rate: #{inclusions_before.to_f/views_before}"

    pp "AFTER"
    pp "Views: #{views_after}"
    pp "Comments: #{comments_after}"
    pp "Rate: #{comments_after.to_f/views_after}"

    pp "Inclusions: #{inclusions_after}"
    pp "Rate: #{inclusions_after.to_f/views_after}"

  end

  task :fact_checking => :environment do 
    puts "Fact-checking metrics"


    impacts = []
    [0,1,2].each do |verdict|
      puts "For points with an overall " + verdict.to_s + " verdict"
      col = :overall_verdict
      #col = :max_verdict
      assessments = Assessable::Assessment.where(col => verdict)

      inclusions = { :before => 0, :after => 0 } 
      views = { :before => 0, :after => 0 }

      normalized_impact = 0
      cnt = 0

      assessments.each do |ass|
        pivot = ass.updated_at
        point = ass.root_object

        #next if ass.claims.count > 1
        #next if point.point_listings.where('created_at > "' + pivot.to_s + '"').count < 10


        population_before = point.proposal.inclusions.where('created_at < "' + pivot.to_s + '"').count.to_f / point.proposal.point_listings.where('created_at < "' + pivot.to_s + '"').count
        population_after = point.proposal.inclusions.where('created_at > "' + pivot.to_s + '"').count.to_f / point.proposal.point_listings.where('created_at > "' + pivot.to_s + '"').count
        population_after = population_after.infinite? ? 1.0 : population_after

        point_before = point.inclusions.where('created_at < "' + pivot.to_s + '"').count.to_f / point.point_listings.where('created_at < "' + pivot.to_s + '"').count
        point_after = point.inclusions.where('created_at > "' + pivot.to_s + '"').count.to_f / point.point_listings.where('created_at > "' + pivot.to_s + '"').count
        point_after = point_after.infinite? ? 1.0 : point_after

        next if point_after.nan?

        normalized_impact += (point_before > 0 ? point_after / point_before : 0) - (population_before > 0 ? population_after / population_before : 0)
        inclusions[:before] += point.inclusions.where('created_at < "' + pivot.to_s + '"').count
        inclusions[:after] += point.inclusions.where('created_at > "' + pivot.to_s + '"') .count
        views[:before] += point.point_listings.where('created_at < "' + pivot.to_s + '"').count
        views[:after] += point.point_listings.where('created_at > "' + pivot.to_s + '"').count
        cnt += 1

        impacts.push([point, normalized_impact, cnt, point.point_listings.where('created_at > "' + pivot.to_s + '"').count, verdict])

      end

      printf("%s:\tinclusions %i\tviews %i\t%.3f\n", 'Before', inclusions[:before], views[:before], inclusions[:before].to_f / views[:before] )
      printf("%s:\tinclusions %i\tviews %i\t%.3f\n", 'After', inclusions[:after], views[:after], inclusions[:after].to_f / views[:after] )

      printf("Impact:\t%.3f\n", (inclusions[:after].to_f / views[:after]) / (inclusions[:before].to_f / views[:before]) )
      printf("Normalized:\t%.3f\n", normalized_impact / cnt )
      pp cnt


    end

    impacts.sort! {|a,b| a[1] <=> b[1]}

    impacts.each do |i|
      printf("%i\t%i\t%i\t%.3f\t%i\n", i[4], i[0].point_listings.count, i[3], i[1], i[0].id)
    end
  end

  task :fact_checking_discussion => :environment do 
    puts "Fact-checking metrics"


    impacts = []
    proposal_point_map = {}
    Point.where('comment_count > 0').each do |pnt|
      comts = pnt.comments.map {|x| x.created_at}.compact
      if !proposal_point_map.has_key? pnt.proposal.id
        proposal_point_map[pnt.proposal_id] = [comts]
      else
        proposal_point_map[pnt.proposal_id].push comts
      end
    end
    #pp proposal_point_map

    [0,1,2].each do |verdict|
      puts "For points with an overall " + verdict.to_s + " verdict"
      col = :overall_verdict
      #col = :max_verdict
      assessments = Assessable::Assessment.where(col => verdict)

      comments = { :before => 0, :after => 0 } 
      views = { :before => 0, :after => 0 }

      normalized_impact = 0
      cnt = 0


      assessments.each do |ass|
        pivot = ass.updated_at
        point = ass.root_object

        #next if point.comments.count == 0
        #next if ass.claims.count > 1
        #next if point.point_listings.where('created_at > "' + pivot.to_s + '"').count < 10

        comments_before_pivot = comments_after_pivot = 0.0

        proposal_point_map[point.proposal_id].each do |pnt|
          pnt.each do |comment|
            if comment < pivot
              comments_before_pivot += 1
            else  
              comments_after_pivot += 1
            end
          end 
        end

        point_before = point.comments.where('created_at < "' + pivot.to_s + '"').count.to_f / point.point_listings.where('created_at < "' + pivot.to_s + '"').count
        point_after = point.comments.where('created_at > "' + pivot.to_s + '"').count.to_f / point.point_listings.where('created_at > "' + pivot.to_s + '"').count
        point_after = point_after.infinite? ? 1.0 : point_after

        population_before = comments_before_pivot / point.proposal.point_listings.where('created_at < "' + pivot.to_s + '"').count
        population_after = comments_after_pivot / point.proposal.point_listings.where('created_at > "' + pivot.to_s + '"').count
        population_after = population_after.infinite? ? 1.0 : population_after

        
        next if point_after.nan?

        normalized_impact += (point_before > 0 ? point_after / point_before : 0) - (population_before > 0 ? population_after / population_before : 0)
        comments[:before] += point.comments.where('created_at < "' + pivot.to_s + '"').count
        comments[:after] += point.comments.where('created_at > "' + pivot.to_s + '"') .count
        views[:before] += point.point_listings.where('created_at < "' + pivot.to_s + '"').count
        views[:after] += point.point_listings.where('created_at > "' + pivot.to_s + '"').count
        cnt += 1

        impacts.push([point, normalized_impact, cnt, point.point_listings.where('created_at > "' + pivot.to_s + '"').count, verdict])

      end

      printf("%s:\tcomments %i\tviews %i\t%.3f\n", 'Before', comments[:before], views[:before], comments[:before].to_f / views[:before] )
      printf("%s:\tcomments %i\tviews %i\t%.3f\n", 'After', comments[:after], views[:after], comments[:after].to_f / views[:after] )

      printf("Impact:\t%.3f\n", (comments[:after].to_f / views[:after]) / (comments[:before].to_f / views[:before]) )
      printf("Normalized:\t%.3f\n", normalized_impact / cnt )
      pp cnt


    end

    impacts.sort! {|a,b| a[1] <=> b[1]}

    impacts.each do |i|
      printf("%i\t%i\t%i\t%.3f\t%i\n", i[4], i[0].point_listings.count, i[3], i[1], i[0].id)
    end
  end

  task :export_fact_checking => :environment do 

    election_date = DateTime.new(2012,11,8)

    Assessable::Assessment.all().each do |ass|
      #pp "Created at: #{ass.created_at}, Updated at: #{ass.updated_at}"
      #pp "requested at: #{ass.requests.first.created_at}"
    end
    # average_date = Assessable::Assessment.select('FROM_UNIXTIME(AVG(UNIX_TIMESTAMP(created_at) + (UNIX_TIMESTAMP(updated_at) - UNIX_TIMESTAMP(created_at))/2 )) as updated_at')
    # average_date = average_date[0].updated_at

    points = Assessable::Assessment.select('distinct(assessable_id)').map {|a| a.assessable_id }.compact

    proposals = points.map {|p| Point.find(p).proposal_id}.compact.uniq
    all_points = Point.where("proposal_id in (?) AND created_at < (?)", proposals, election_date)

    nf_inclusions_all = Inclusion.where("proposal_id in (?) and point_id not in (?)  AND created_at < (?)", proposals, points, election_date)

    nf_all_points = Point.where("proposal_id in (?)", proposals)
    is_pros = {}
    nf_all_points.each do |p|
      is_pros[p.id] = p
    end
    nf_comments_all = Commentable::Comment.where("commentable_id in (?) and commentable_id not in (?) AND created_at < (?)", all_points.map {|p| p.id}.compact, points, election_date)

    nf_all_views = PointListing.where("point_id in (?) and point_id not in (?) and created_at is not null  AND created_at < (?)", all_points.map {|p| p.id}.compact, points, election_date)

    fc_inclusions_all = Inclusion.where("point_id in (?) AND created_at < (?)", points, election_date)

    fc_comments_all = Commentable::Comment.where("commentable_id in (?) AND created_at < (?)", points, election_date)

    fc_all_views = PointListing.where("point_id in (?) and created_at is not null AND created_at < (?)", points, election_date)

    tmp = {}
    nf_all_views.each do |vw|
      if !tmp.has_key? [vw.point_id, vw.user_id]
        tmp[[vw.point_id,vw.user_id]] = vw
      end
    end
    nf_all_views = tmp.values()

    tmp = {}
    fc_all_views.each do |vw|
      if !tmp.has_key? [vw.point_id, vw.user_id]
        tmp[[vw.point_id,vw.user_id]] = vw
      end
    end
    fc_all_views = tmp.values()


    views_before = 0
    views_after = 0

    inclusions_before = 0
    inclusions_after = 0

    comments_after = 0
    comments_before = 0

    views_before0 = 0
    views_before1 = 0
    views_before2 = 0

    views_after0 = 0
    views_after1 = 0
    views_after2 = 0

    inclusions_before0 = 0
    inclusions_before1 = 0
    inclusions_before2 = 0

    inclusions_after0 = 0
    inclusions_after1 = 0
    inclusions_after2 = 0

    simulate_fact_check = true
    simulated_date = DateTime.new(2012,10,28)
    simulated_assessment = Assessable::Assessment.new(:updated_at => simulated_date, :overall_verdict => -1)

    require 'csv'
    CSV.open("data.csv", "w") do |csv|

      head = ['firstview_timestamp', 'point_id', 'initiative_id', 'is_pro',  'was_point_factchecked', 'factchecked_before_view', 'fact_check_accuracy_verdict', 'fact_check_timestamp', 'user_commented', 'user_commented_before_factcheck', 'user_commented_after_factcheck', 'user_included_point']


      csv << head
      [ [fc_all_views, true], [nf_all_views, false] ].each do |views, was_checked|
        views.each do |view|
          row = []

          # Time stamp for the view
          row.push view.created_at

          # An identifier for the point that was viewed
          row.push view.point_id

          #initiative_id
          row.push view.proposal_id

          #is_pro
          row.push is_pros[view.point_id].is_pro

          # An indicator for whether or not the point was ever fact checked (regardless of whether the particular view occurred before or after the fact check)
          row.push was_checked

          # An indicator for whether or not the point had been fact checked prior to the view
          assessment = Assessable::Assessment.find_by_assessable_id(view.point_id)
          if simulate_fact_check && !was_checked
            assessment = simulated_assessment
          end

          row.push was_checked || simulate_fact_check ? assessment.updated_at > view.created_at : nil
{'fraction_non_alphanumeric': 0.14187433439829605, 'fraction_numerical': 0.011501597444089457, 'mean_word_length': 2.9795728089506697, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 241, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '642620', 'n_tokens_mistral': 16396, 'n_tokens_neox': 15622, 'n_words': 2828}