<commit_before># encoding: utf-8
TRS_FILE_PATH = "reports/trsystem.txt";

namespace :trsystem do

  #Usage
  #Development:   bundle exec rake trsystem:all
  #In production: bundle exec rake trsystem:all RAILS_ENV=production
  task :all => :environment do
    Rake::Task["trsystem:prepare"].invoke
    Rake::Task["trsystem:usage"].invoke(false)
    Rake::Task["trsystem:rs"].invoke(false)
    Rake::Task["trsystem:rsViSH"].invoke(false)
  end

  task :prepare do
    require "#{Rails.root}/lib/task_utils"
    prepareFile(TRS_FILE_PATH)
    writeInTRS("Tracking System Report")
  end

  task :usage, [:prepare] => :environment do |t,args|
    args.with_defaults(:prepare => true)

    if args.prepare
      Rake::Task["trsystem:prepare"].invoke
    end

    writeInTRS("")
    writeInTRS("Usage Report")
    writeInTRS("")

    vvEntries = TrackingSystemEntry.where(:app_id=>"ViSH Viewer")

    #Time period
    #Entries of the last 3 months
    # endDate = Time.now
    # startDate = endDate.advance(:months => -3)
    # vvEntries = vvEntries.where(:created_at => startDate..endDate)

    totalSamples = 0
    totalSlides = 0
    totalDuration = 0

    vvEntries.each do |e|
      # begin
        d = JSON(e["data"]) rescue {}

        unless d["chronology"].nil? or d["duration"].nil?
          # chronologyEntries = d["chronology"].values
          nSlides = d["chronology"].values.map{|c| c["slideNumber"]}.uniq.length
          # tDuration = chronologyEntries.map{|c| c["duration"].to_f}.sum
          tDuration = d["duration"].to_f

          #Filter extremely long and short durations
          MAX_DURATION = 1.5*60*60 #1.5 hours
          MIN_DURATION = 10 #10 secs

          if nSlides.is_a? Integer and tDuration.is_a? Float and nSlides>0 and tDuration>MIN_DURATION and tDuration<MAX_DURATION
            totalSlides += nSlides
            totalDuration += tDuration
            totalSamples += 1
          end
        end
      # rescue
      # end
    end

    writeInTRS("Total samples")
    writeInTRS(totalSamples)
    writeInTRS("Average time per slide:")
    if totalDuration > 0
      writeInTRS((totalDuration/totalSlides.to_f).round(2).to_s + " (s)")
    else
      writeInTRS("0 (s)")
    end

  end

  #Usage
  #Development:   bundle exec rake trsystem:rs
  #In production: bundle exec rake trsystem:rs RAILS_ENV=production
  task :rs, [:prepare] => :environment do |t,args|
    args.with_defaults(:prepare => true)

    if args.prepare
      Rake::Task["trsystem:prepare"].invoke
    end

    writeInTRS("")
    writeInTRS("Recommender System Report. ViSH Viewer data")
    writeInTRS("")

    vvEntries = TrackingSystemEntry.where(:app_id=>"ViSH Viewer")

    recTSD = {}

    recTSD["Random"] = {}
    recTSD["Random"]["totalRec"] = 0
    recTSD["Random"]["totalRecShow"] = 0
    recTSD["Random"]["totalRecAccepted"] = 0
    recTSD["Random"]["totalRecDenied"] = 0
    recTSD["Random"]["timeToAccept"] = []

    recTSD["ViSHRecommenderSystem"] = {}
    recTSD["ViSHRecommenderSystem"]["totalRec"] = 0
    recTSD["ViSHRecommenderSystem"]["totalRecShow"] = 0
    recTSD["ViSHRecommenderSystem"]["totalRecAccepted"] = 0
    recTSD["ViSHRecommenderSystem"]["totalRecDenied"] = 0
    recTSD["ViSHRecommenderSystem"]["timeToAccept"] = []

    #Compare Accepted group vs Denied group
    recTSD["ViSHRecommenderSystem"]["accepted"] = []
    recTSD["ViSHRecommenderSystem"]["denied"] = []


    vvEntries.each do |e|
      d = JSON(e["data"]) rescue {}
      recData = d["rs"]
      unless recData.nil? or !recData["tdata"].is_a? Hash
        firstItem = recData["tdata"].values.first
        rsItemTrackingData = JSON(firstItem["recommender_data"]) rescue nil
        unless rsItemTrackingData.nil? or !["Random","ViSHRecommenderSystem"].include? rsItemTrackingData["rec"]
          thisRecTSD = recTSD[rsItemTrackingData["rec"]]
          thisRecTSD["totalRec"] += 1

          if recData["shown"]=="true" || recData["shown"]==true
            thisRecTSD["totalRecShow"] += 1
          end

          if recData["accepted"] == "false" or recData["accepted"]==false
            thisRecTSD["totalRecDenied"] += 1
          elsif recData["accepted"] == "undefined"
            #Do nothing
          elsif recData["accepted"].is_a? String
            thisRecTSD["totalRecAccepted"] += 1

            #When accepted, measure time.
            allActions = d["chronology"].values.map{|c| c["actions"].values}.flatten rescue []
            onShowRecommendationAction = allActions.select{|a| a["id"]=="onShowRecommendations" }.last
            onAcceptRecommendationAction = allActions.select{|a| a["id"]=="onAcceptRecommendation" }.last

            if !onShowRecommendationAction.nil? and !onAcceptRecommendationAction.nil? and !onShowRecommendationAction["t"].nil? and !onAcceptRecommendationAction["t"].nil?
              recTime = (onAcceptRecommendationAction["t"].to_f - onShowRecommendationAction["t"].to_f).round(2)
              if recTime > 0
                thisRecTSD["timeToAccept"].push(recTime)
              end
            end

            #When accepted, and RS is ViSHRecommender, store accepted and denied items
            if rsItemTrackingData["rec"] == "ViSHRecommenderSystem"
              acceptedItem = recData["tdata"].values.select{|item| item["id"]==recData["accepted"]}[0]
              acceptedItemData = JSON(acceptedItem["recommender_data"])
              recTSD["ViSHRecommenderSystem"]["accepted"].push(acceptedItemData)

              deniedItemsLength = recData["tdata"].values.select{|item| item["id"]!=recData["accepted"]}
              deniedItemsLengthData = deniedItemsLength.map{|item| JSON(item["recommender_data"])}
              recTSD["ViSHRecommenderSystem"]["denied"] += deniedItemsLengthData
            end
          end
        end
      end
    end


    ###############
    # ViSH Recommender System vs Random
    ###############

    if recTSD["Random"]["timeToAccept"].length > 0
      randomAverageTimeToAccept = (recTSD["Random"]["timeToAccept"].sum/recTSD["Random"]["timeToAccept"].size.to_f).round(2)
    else
      randomAverageTimeToAccept = 0
    end

    if recTSD["ViSHRecommenderSystem"]["timeToAccept"].length > 0
      vishRSAverageTimeToAccept = (recTSD["ViSHRecommenderSystem"]["timeToAccept"].sum/recTSD["ViSHRecommenderSystem"]["timeToAccept"].size.to_f).round(2)
    else
      vishRSAverageTimeToAccept = 0
    end

    writeInTRS("")
    writeInTRS("Recommender System: Random")
    writeInTRS("Showed Recommendations:")
    writeInTRS(recTSD["Random"]["totalRecShow"])
    writeInTRS("Accepted Recommendations:")
    writeInTRS(recTSD["Random"]["totalRecAccepted"])
    writeInTRS("Denied Recommendations:")
    writeInTRS(recTSD["Random"]["totalRecDenied"])
    writeInTRS("Average time to accept a recommendation:")
    writeInTRS(randomAverageTimeToAccept)
    
    writeInTRS("")
    writeInTRS("Recommender System: ViSH Recommender")
    writeInTRS("Showed Recommendations:")
    writeInTRS(recTSD["ViSHRecommenderSystem"]["totalRecShow"])
    writeInTRS("Accepted Recommendations:")
    writeInTRS(recTSD["ViSHRecommenderSystem"]["totalRecAccepted"])
    writeInTRS("Denied Recommendations:")
    writeInTRS(recTSD["ViSHRecommenderSystem"]["totalRecDenied"])
    writeInTRS("Average time to accept a recommendation:")
    writeInTRS(vishRSAverageTimeToAccept)


    ###############
    # Accepted vs denied LOs
    ###############

    acceptedItemsLength = [1,recTSD["ViSHRecommenderSystem"]["accepted"].length].max
    deniedItemsLength = [1,recTSD["ViSHRecommenderSystem"]["denied"].length].max

    accceptedAverageOverallScore = (recTSD["ViSHRecommenderSystem"]["accepted"].map{|i| i["overall_score"]}.sum/acceptedItemsLength.to_f).round(4)
    deniedAverageOverallScore = (recTSD["ViSHRecommenderSystem"]["denied"].map{|i| i["overall_score"]}.sum/deniedItemsLength.to_f).round(4)

    accceptedAverageCSScore = (recTSD["ViSHRecommenderSystem"]["accepted"].map{|i| i["cs_score"]}.sum/acceptedItemsLength.to_f).round(4)
    deniedAverageCSScore = (recTSD["ViSHRecommenderSystem"]["denied"].map{|i| i["cs_score"]}.sum/deniedItemsLength.to_f).round(4)

    accceptedAverageUSScore = (recTSD["ViSHRecommenderSystem"]["accepted"].reject{|i| i["us_score"].nil?}.map{|i| i["us_score"]}.sum/acceptedItemsLength.to_f).round(4)
    deniedAverageUSScore = (recTSD["ViSHRecommenderSystem"]["denied"].reject{|i| i["us_score"].nil?}.map{|i| i["us_score"]}.sum/deniedItemsLength.to_f).round(4)

    accceptedAveragePopularityScore = (recTSD["ViSHRecommenderSystem"]["accepted"].map{|i| i["popularity_score"]}.sum/acceptedItemsLength.to_f).round(4)
    deniedAveragePopularityScore = (recTSD["ViSHRecommenderSystem"]["denied"].map{|i| i["popularity_score"]}.sum/deniedItemsLength.to_f).round(4)

    accceptedAverageQualityScore = (recTSD["ViSHRecommenderSystem"]["accepted"].map{|i| i["quality_score"]}.sum/acceptedItemsLength.to_f).round(4)
    deniedAverageQualityScore = (recTSD["ViSHRecommenderSystem"]["denied"].map{|i| i["quality_score"]}.sum/deniedItemsLength.to_f).round(4)

    writeInTRS("")
    writeInTRS("Group of accepted LOs")
    writeInTRS("Overall score:")
    writeInTRS(accceptedAverageOverallScore)
    writeInTRS("Content similarity score:")
    writeInTRS(accceptedAverageCSScore)
    writeInTRS("User similarity score:")
    writeInTRS(accceptedAverageUSScore)
    writeInTRS("Popularity score:")
    writeInTRS(accceptedAveragePopularityScore)
    writeInTRS("Quality score:")
    writeInTRS(accceptedAverageQualityScore)

    writeInTRS("")
    writeInTRS("Group of denied LOs")
    writeInTRS("Overall score:")
    writeInTRS(deniedAverageOverallScore)
    writeInTRS("Content similarity score:")
    writeInTRS(deniedAverageCSScore)
    writeInTRS("User similarity score:")
    writeInTRS(deniedAverageUSScore)<commit_msg>Fix bug on Tracker RS Analytics<commit_after># encoding: utf-8
TRS_FILE_PATH = "reports/trsystem.txt";

namespace :trsystem do

  #Usage
  #Development:   bundle exec rake trsystem:all
  #In production: bundle exec rake trsystem:all RAILS_ENV=production
  task :all => :environment do
    Rake::Task["trsystem:prepare"].invoke
    Rake::Task["trsystem:usage"].invoke(false)
    Rake::Task["trsystem:rs"].invoke(false)
    Rake::Task["trsystem:rsViSH"].invoke(false)
  end

  task :prepare do
    require "#{Rails.root}/lib/task_utils"
    prepareFile(TRS_FILE_PATH)
    writeInTRS("Tracking System Report")
  end

  task :usage, [:prepare] => :environment do |t,args|
    args.with_defaults(:prepare => true)

    if args.prepare
      Rake::Task["trsystem:prepare"].invoke
    end

    writeInTRS("")
    writeInTRS("Usage Report")
    writeInTRS("")

    vvEntries = TrackingSystemEntry.where(:app_id=>"ViSH Viewer")

    #Time period
    #Entries of the last 3 months
    # endDate = Time.now
    # startDate = endDate.advance(:months => -3)
    # vvEntries = vvEntries.where(:created_at => startDate..endDate)

    totalSamples = 0
    totalSlides = 0
    totalDuration = 0

    vvEntries.each do |e|
      # begin
        d = JSON(e["data"]) rescue {}

        unless d["chronology"].nil? or d["duration"].nil?
          # chronologyEntries = d["chronology"].values
          nSlides = d["chronology"].values.map{|c| c["slideNumber"]}.uniq.length
          # tDuration = chronologyEntries.map{|c| c["duration"].to_f}.sum
          tDuration = d["duration"].to_f

          #Filter extremely long and short durations
          MAX_DURATION = 1.5*60*60 #1.5 hours
          MIN_DURATION = 10 #10 secs

          if nSlides.is_a? Integer and tDuration.is_a? Float and nSlides>0 and tDuration>MIN_DURATION and tDuration<MAX_DURATION
            totalSlides += nSlides
            totalDuration += tDuration
            totalSamples += 1
          end
        end
      # rescue
      # end
    end

    writeInTRS("Total samples")
    writeInTRS(totalSamples)
    writeInTRS("Average time per slide:")
    if totalDuration > 0
      writeInTRS((totalDuration/totalSlides.to_f).round(2).to_s + " (s)")
    else
      writeInTRS("0 (s)")
    end

  end

  #Usage
  #Development:   bundle exec rake trsystem:rs
  #In production: bundle exec rake trsystem:rs RAILS_ENV=production
  task :rs, [:prepare] => :environment do |t,args|
    args.with_defaults(:prepare => true)

    if args.prepare
      Rake::Task["trsystem:prepare"].invoke
    end

    writeInTRS("")
    writeInTRS("Recommender System Report. ViSH Viewer data")
    writeInTRS("")

    vvEntries = TrackingSystemEntry.where(:app_id=>"ViSH Viewer")

    recTSD = {}

    recTSD["Random"] = {}
    recTSD["Random"]["totalRec"] = 0
    recTSD["Random"]["totalRecShow"] = 0
    recTSD["Random"]["totalRecAccepted"] = 0
    recTSD["Random"]["totalRecDenied"] = 0
    recTSD["Random"]["timeToAccept"] = []

    recTSD["ViSHRecommenderSystem"] = {}
    recTSD["ViSHRecommenderSystem"]["totalRec"] = 0
    recTSD["ViSHRecommenderSystem"]["totalRecShow"] = 0
    recTSD["ViSHRecommenderSystem"]["totalRecAccepted"] = 0
    recTSD["ViSHRecommenderSystem"]["totalRecDenied"] = 0
    recTSD["ViSHRecommenderSystem"]["timeToAccept"] = []

    #Compare Accepted group vs Denied group
    recTSD["ViSHRecommenderSystem"]["accepted"] = []
    recTSD["ViSHRecommenderSystem"]["denied"] = []


    vvEntries.each do |e|
      d = JSON(e["data"]) rescue {}
      recData = d["rs"]
      unless recData.nil? or !recData["tdata"].is_a? Hash
        firstItem = recData["tdata"].values.first
        rsItemTrackingData = JSON(firstItem["recommender_data"]) rescue nil
        unless rsItemTrackingData.nil? or !["Random","ViSHRecommenderSystem"].include? rsItemTrackingData["rec"]
          thisRecTSD = recTSD[rsItemTrackingData["rec"]]
          thisRecTSD["totalRec"] += 1

          if recData["shown"]=="true" || recData["shown"]==true
            thisRecTSD["totalRecShow"] += 1
          end

          if recData["accepted"] == "false" or recData["accepted"]==false
            thisRecTSD["totalRecDenied"] += 1
          elsif recData["accepted"] == "undefined"
            #Do nothing
          elsif recData["accepted"].is_a? String
            thisRecTSD["totalRecAccepted"] += 1

            #When accepted, measure time.
            allActions = d["chronology"].values.map{|c| c["actions"].values}.flatten rescue []
            onShowRecommendationAction = allActions.select{|a| a["id"]=="onShowRecommendations" }.last
            onAcceptRecommendationAction = allActions.select{|a| a["id"]=="onAcceptRecommendation" }.last

            if !onShowRecommendationAction.nil? and !onAcceptRecommendationAction.nil? and !onShowRecommendationAction["t"].nil? and !onAcceptRecommendationAction["t"].nil?
              recTime = (onAcceptRecommendationAction["t"].to_f - onShowRecommendationAction["t"].to_f).round(2)
              if recTime > 0
                thisRecTSD["timeToAccept"].push(recTime)
              end
            end

            #When accepted, and RS is ViSHRecommender, store accepted and denied items
            if rsItemTrackingData["rec"] == "ViSHRecommenderSystem"
              acceptedItem = recData["tdata"].values.select{|item| item["id"]==recData["accepted"]}[0]
              recTSD["ViSHRecommenderSystem"]["accepted"].push(acceptedItem)
              deniedItems = recData["tdata"].values.select{|item| item["id"]!=recData["accepted"]}
              recTSD["ViSHRecommenderSystem"]["denied"] += deniedItems
            end
          end
        end
      end
    end


    ###############
    # ViSH Recommender System vs Random vs Other recommender approaches
    ###############

    if recTSD["Random"]["timeToAccept"].length > 0
      randomAverageTimeToAccept = (recTSD["Random"]["timeToAccept"].sum/recTSD["Random"]["timeToAccept"].size.to_f).round(2)
    else
      randomAverageTimeToAccept = 0
    end

    if recTSD["ViSHRecommenderSystem"]["timeToAccept"].length > 0
      vishRSAverageTimeToAccept = (recTSD["ViSHRecommenderSystem"]["timeToAccept"].sum/recTSD["ViSHRecommenderSystem"]["timeToAccept"].size.to_f).round(2)
    else
      vishRSAverageTimeToAccept = 0
    end

    writeInTRS("")
    writeInTRS("Recommender System: Random")
    writeInTRS("Showed Recommendations:")
    writeInTRS(recTSD["Random"]["totalRecShow"])
    writeInTRS("Accepted Recommendations:")
    writeInTRS(recTSD["Random"]["totalRecAccepted"])
    writeInTRS("Denied Recommendations:")
    writeInTRS(recTSD["Random"]["totalRecDenied"])
    writeInTRS("Average time to accept a recommendation:")
    writeInTRS(randomAverageTimeToAccept)
    
    writeInTRS("")
    writeInTRS("Recommender System: ViSH Recommender")
    writeInTRS("Showed Recommendations:")
    writeInTRS(recTSD["ViSHRecommenderSystem"]["totalRecShow"])
    writeInTRS("Accepted Recommendations:")
    writeInTRS(recTSD["ViSHRecommenderSystem"]["totalRecAccepted"])
    writeInTRS("Denied Recommendations:")
    writeInTRS(recTSD["ViSHRecommenderSystem"]["totalRecDenied"])
    writeInTRS("Average time to accept a recommendation:")
    writeInTRS(vishRSAverageTimeToAccept)


    ###############
    # Accepted vs denied LOs
    ###############

    acceptedItemsLength = [1,recTSD["ViSHRecommenderSystem"]["accepted"].length].max
    deniedItemsLength = [1,recTSD["ViSHRecommenderSystem"]["denied"].length].max

    accceptedQualityScores = recTSD["ViSHRecommenderSystem"]["accepted"].map{ |i| 
      qscore = nil
      recData = JSON(i["recommender_data"]) rescue {}
      unless recData["qscore"].nil?
        qscore = recData["qscore"]
      else
        excursion = Excursion.find_by_id(i["id"])
        unless excursion.nil?
          qscore = excursion.qscore
        end
      end
      qscore
    }.compact

    deniedQualityScores = recTSD["ViSHRecommenderSystem"]["denied"].map{ |i| 
      qscore = nil
      recData = JSON(i["recommender_data"]) rescue {}
      unless recData["qscore"].nil?
        qscore = recData["qscore"]
      else
        excursion = Excursion.find_by_id(i["id"])
        unless excursion.nil?
          qscore = excursion.qscore
        end
      end
      qscore
    }.compact

    accceptedPopularityScores = recTSD["ViSHRecommenderSystem"]["accepted"].map{ |i| 
      popularity = nil
      recData = JSON(i["recommender_data"]) rescue {}
      unless recData["popularity"].nil?
        popularity = recData["popularity"]
      else
        excursion = Excursion.find_by_id(i["id"])
        unless excursion.nil?
          popularity = excursion.popularity
        end
      end
      popularity
    }.compact

    deniedPopularityScores = recTSD["ViSHRecommenderSystem"]["denied"].map{ |i| 
      popularity = nil
      recData = JSON(i["recommender_data"]) rescue {}
      unless recData["popularity"].nil?
        popularity = recData["popularity"]
      else
        excursion = Excursion.find_by_id(i["id"])
        unless excursion.nil?
          popularity = excursion.popularity
        end
      end
      popularity
    }.compact

    accceptedAverageOverallScore = (recTSD["ViSHRecommenderSystem"]["accepted"].map{|i| JSON(i["recommender_data"])["overall_score"]}.sum/acceptedItemsLength.to_f).round(4)
    deniedAverageOverallScore = (recTSD["ViSHRecommenderSystem"]["denied"].map{|i| JSON(i["recommender_data"])["overall_score"]}.sum/deniedItemsLength.to_f).round(4)

    accceptedAverageCSScore = (recTSD["ViSHRecommenderSystem"]["accepted"].map{|i| JSON(i["recommender_data"])["cs_score"]}.sum/acceptedItemsLength.to_f).round(4)
    deniedAverageCSScore = (recTSD["ViSHRecommenderSystem"]["denied"].map{|i| JSON(i["recommender_data"])["cs_score"]}.sum/deniedItemsLength.to_f).round(4)

    accceptedAverageUSScore = (recTSD["ViSHRecommenderSystem"]["accepted"].reject{|i| JSON(i["recommender_data"])["us_score"].nil?}.map{|i| JSON(i["recommender_data"])["us_score"]}.sum/acceptedItemsLength.to_f).round(4)
    deniedAverageUSScore = (recTSD["ViSHRecommenderSystem"]["denied"].reject{|i| JSON(i["recommender_data"])["us_score"].nil?}.map{|i| JSON(i["recommender_data"])["us_score"]}.sum/deniedItemsLength.to_f).round(4)

    accceptedAveragePopularityScore = ((accceptedPopularityScores.sum/([1,accceptedPopularityScores.length].max)).to_f).round(0)
    deniedAveragePopularityScore = ((deniedPopularityScores.sum/([1,deniedPopularityScores.length].max)).to_f).round(0)

    accceptedAverageQualityScore = ((accceptedQualityScores.sum/([1,accceptedQualityScores.length].max)).to_f).round(0)
    deniedAverageQualityScore = ((deniedQualityScores.sum/([1,deniedQualityScores.length].max)).to_f).round(0)

    writeInTRS("")
    writeInTRS("Group of accepted LOs")
    writeInTRS("Overall score:")
    writeInTRS(accceptedAverageOverallScore)
    writeInTRS("Content similarity score:")
    writeInTRS(accceptedAverageCSScore)
    writeInTRS("User similarity score:")
    writeInTRS(accceptedAverageUSScore)
    writeInTRS("Popularity score:")
    writeInTRS(accceptedAveragePopularityScore)
    writeInTRS("Quality score:")
    writeInTRS(accceptedAverageQualityScore)

    writeInTRS("")
    writeInTRS("Group of denied LOs")
    writeInTRS("Overall score:")
    writeInTRS(deniedAverageOverallScore)
    writeInTRS("Content similarity score:")
    writeInTRS(deniedAverageCSScore)
    writeInTRS("User similarity score:")
    writeInTRS(deniedAverageUSScore)
{'fraction_non_alphanumeric': 0.1432663410969196, 'fraction_numerical': 0.005447032306536439, 'mean_word_length': 3.8902411021814007, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1645475', 'n_tokens_mistral': 7184, 'n_tokens_neox': 6922, 'n_words': 1051}