<commit_before>
desc "Fix all"
task :fixall => [:environment, "fixdb:openid", "fixdb:groups", "fixdb:relocate", "fixdb:counters", "fixdb:sync_counts", "fixdb:votes", "fixdb:questions", "fixdb:comments", "fixdb:update_questions_widgets"] do
end

namespace :fixdb do
  task :openid => [:environment] do
    User.find_each do |user|
      next if user.identity_url.blank?

      puts "Updating: #{user.login}"
      user.push_uniq(:auth_keys => "open_id_#{user[:identity_url]}")
      user.unset(:identity_url => 1)
    end
  end

  task :sync_counts => [:environment] do
    votes = MongoMapper.database.collection("votes")
    comments = MongoMapper.database.collection("comments")
    puts "updating comment's counts"
    comments.find.each do |c|
      print "."
      votes_average=0
      votes.find(:voteable_id =>  c["_id"]).each do |v|
        votes_average+=v["value"]
      end
      comments.update({:_id => c["id"]},
                      {"$set" => {"votes_count" => votes.find(:voteable_id =>  c["_id"]).count,
                                  "votes_average" => votes_average}})

      if c["flags"]
        comments.update({:_id => c["id"]}, {"$set" => {"flags_count" => c["flags"].size}})
      end
    end

    puts "updating questions's counts"
    Question.find_each do |q|
      print "."
      votes_average=0
      q.votes.each {|e| votes_average+=e.value }
      q.set("flags_count" => q.flags.size, "votes_count" => q.votes.size, "votes_average" => votes_average)
    end
  end

  task :counters => :environment do
    Question.find_each do |q|
      q.set(:close_requests_count => q.close_requests.size)
      q.set(:open_requests_count => q.open_requests.size)
    end
  end

  task :questions => [:environment] do
    puts "updating questions#last_target_type"
    Question.find_each(:last_target_type.ne => nil) do |q|
      print "."
      if(q.last_target_type != "Comment")
        last_target = q.last_target_type.constantize.find(q.last_target_id)
      else
        data = MongoMapper.database.collection("comments").find_one(:_id => q.last_target_id)
        last_target = Comment.new(data)
#         p last_target.id
#         p MongoMapper.database.collection("comments").find_one(:_id => q.last_target_id)
      end

      if(last_target)
        if(last_target.respond_to?(:updated_at) && last_target.updated_at && last_target.updated_at.is_a?(String))
          last_target.updated_at = Time.parse(last_target.updated_at)
        end
        Question.update_last_target(q.id, last_target)
      end
    end
  end

  task :votes => [:environment] do
    puts "updating votes"
    Group.find_each do |group|
      count = 0

      comments = MongoMapper.database.collection("comments")
      questions = MongoMapper.database.collection("questions")
      MongoMapper.database.collection("votes").find({:group_id => group["_id"]}).each do |vote|
        vote.delete("group_id")
        id = vote.delete("voteable_id")
        klass = vote.delete("voteable_type")
        collection = comments
        if klass == "Question"
          collection = questions;
        end
        count += 1
        collection.update({:_id => id}, "$addToSet" => {:votes => vote})
      end
      if count > 0
        puts "Updated #{count} #{group["name"]} votes"
      end
    end
    MongoMapper.database.collection("votes").drop
  end

  task :comments => [:environment] do
    puts "updating comments"
    comments = MongoMapper.database.collection("comments")
    questions = MongoMapper.database.collection("questions")

    MongoMapper.database.collection("comments").find(:_type => "Comment").each do |comment|
        id = comment.delete("commentable_id")
        klass = comment.delete("commentable_type")
        collection = comments

        if klass == "Question"
          collection = questions;
        end

        collection.update({:_id => id}, "$addToSet" => {:comments => comment})
    end
    begin
      MongoMapper.database.collection("answers").drop
    ensure
      begin
        comments.rename("answers")
      rescue
        puts "comments collection doesn't exists"
      end
    end
    puts "updated comments"
  end

  task :groups => [:environment] do
    Group.find_each(:language => [nil, '', 'none']) do |group|
      lang = group.description.to_s.language
      puts "Updating #{group.name} subdomain='#{group.subdomain}' detected as: #{lang}"

      group.language = (lang == :spanish) ? 'es' : 'en'
      group.languages = DEFAULT_USER_LANGUAGES
      if group.valid?
        group.save
      else
        puts "Invalid group: #{group.errors.full_messages}"
      end
    end
  end

  task :relocate => [:environment] do
    doc = JSON.parse(File.read('data/countries.json'))
    i=0
    doc.keys.each do |key|
      User.all({ :country_name => key}).each do |u|
        p "#{u.login}: before: #{u.country_name}, after: #{doc[key]["address"]["country"]}"
        lat = doc[key]["lat"]
        lon = doc[key]["lon"]
        User.set({:_id => u.id},
                    {:position => GeoPosition.new(lat, lon).to_mongo,
                      :address => doc[key]["address"]})
#         FIXME
#         Comment.set({:user_id => u.id},
#                     {:position => GeoPosition.new(lat, lon),
#                       :address => doc[key]["address"]})
        Question.set({:user_id => u.id},
                    {:position => GeoPosition.new(lat, lon).to_mongo,
                      :address => doc[key]["address"]})
        Answer.set({:user_id => u.id},
                    {:position => GeoPosition.new(lat, lon).to_mongo,
                      :address => doc[key]["address"]})
      end
    end
  end

  task :update_questions_widgets => [:environment] do
    c=Group.count
    i=1
    Group.all.each do |g|
      [SharingButtonsWidget.new, ModInfoWidget.new, QuestionBadgesWidget.new,
       QuestionStatsWidget.new, QuestionTagsWidget.new, RelatedQuestionsWidget.new,
       TagListWidget.new, CurrentTagsWidget.new].each do |w|
        g.widgets << w
      end
      g.save
      p "(#{i}/#{c}) Updated widgets for group #{g.name}"
      i+=1
    end
  end
end
<commit_msg>Update fixdb rake tasks<commit_after>
desc "Fix all"
task :fixall => [:environment, "fixdb:question_times", "fixdb:openid", "fixdb:groups", "fixdb:relocate", "fixdb:counters", "fixdb:sync_counts", "fixdb:votes", "fixdb:questions", "fixdb:comments", "fixdb:update_questions_widgets"] do
end

namespace :fixdb do
  task :question_times => [:environment] do
    coll = Mongoid.database.collection("questions")
    coll.find.each do |q|
      %w[last_target_date created_at updated_at].each do |key|
        if q[key].is_a?(String)
          q[key] = Time.parse(q[key])
        end
      end
      coll.save(q)
    end
  end

  task :openid => [:environment] do
    User.all.each do |user|
      next if user.identity_url.blank?

      puts "Updating: #{user.login}"
      user.push_uniq(:auth_keys => "open_id_#{user[:identity_url]}")
      user.unset(:identity_url => 1)
    end
  end

  task :sync_counts => [:environment] do
    votes = Mongoid.database.collection("votes")
    comments = Mongoid.database.collection("comments")
    puts "updating comment's counts"
    comments.find.each do |c|
      print "."
      votes_average=0
      votes.find(:voteable_id =>  c["_id"]).each do |v|
        votes_average+=v["value"]
      end
      comments.update({:_id => c["id"]},
                      {"$set" => {"votes_count" => votes.find(:voteable_id =>  c["_id"]).count,
                                  "votes_average" => votes_average}})

      if c["flags"]
        comments.update({:_id => c["id"]}, {"$set" => {"flags_count" => c["flags"].size}})
      end
    end

    puts "updating questions's counts"
    Question.all.each do |q|
      print "."
      votes_average=0
      q.votes.each {|e| votes_average+=e.value }
      q.override("flags_count" => q.flags.size, "votes_count" => q.votes.size, "votes_average" => votes_average)
    end
  end

  task :counters => :environment do
    Question.all.each do |q|
      q.override(:close_requests_count => q.close_requests.size)
      q.override(:open_requests_count => q.open_requests.size)
    end
  end

  task :questions => [:environment] do
    puts "updating questions#last_target_type"
    Question.all(:conditions => {:last_target_type.ne => nil}).each do |q|
      print "."
      if(q.last_target_type != "Comment")
        last_target = q.last_target_type.constantize.find(q.last_target_id)
      else
        data = Mongoid.database.collection("comments").find_one(:_id => q.last_target_id)
        last_target = Comment.new(data)
#         p last_target.id
#         p Mongoid.database.collection("comments").find_one(:_id => q.last_target_id)
      end

      if(last_target)
        if(last_target.respond_to?(:updated_at) && last_target.updated_at && last_target.updated_at.is_a?(String))
          last_target.updated_at = Time.parse(last_target.updated_at)
        end
        Question.update_last_target(q.id, last_target)
      end
    end
  end

  task :votes => [:environment] do
    puts "updating votes"
    Group.all.each do |group|
      count = 0

      comments = Mongoid.database.collection("comments")
      questions = Mongoid.database.collection("questions")
      Mongoid.database.collection("votes").find({:group_id => group["_id"]}).each do |vote|
        vote.delete("group_id")
        id = vote.delete("voteable_id")
        klass = vote.delete("voteable_type")
        collection = comments
        if klass == "Question"
          collection = questions;
        end
        count += 1
        collection.update({:_id => id}, "$addToSet" => {:votes => vote})
      end
      if count > 0
        puts "Updated #{count} #{group["name"]} votes"
      end
    end
    Mongoid.database.collection("votes").drop
  end

  task :comments => [:environment] do
    puts "updating comments"
    comments = Mongoid.database.collection("comments")
    questions = Mongoid.database.collection("questions")

    Mongoid.database.collection("comments").find(:_type => "Comment").each do |comment|
        id = comment.delete("commentable_id")
        klass = comment.delete("commentable_type")
        collection = comments

        if klass == "Question"
          collection = questions;
        end

        collection.update({:_id => id}, "$addToSet" => {:comments => comment})
    end
    begin
      Mongoid.database.collection("answers").drop
    ensure
      begin
        comments.rename("answers")
      rescue
        puts "comments collection doesn't exists"
      end
    end
    puts "updated comments"
  end

  task :groups => [:environment] do
    Group.all(:conditions => {:language.in => [nil, '', 'none']}).each do |group|
      lang = group.description.to_s.language
      puts "Updating #{group.name} subdomain='#{group.subdomain}' detected as: #{lang}"

      group.language = (lang == :spanish) ? 'es' : 'en'
      group.languages = DEFAULT_USER_LANGUAGES
      if group.valid?
        group.save
      else
        puts "Invalid group: #{group.errors.full_messages}"
      end
    end
  end

  task :relocate => [:environment] do
    doc = JSON.parse(File.read('data/countries.json'))
    i=0
    doc.keys.each do |key|
      User.all(:conditions => { :country_name => key}).each do |u|
        p "#{u.login}: before: #{u.country_name}, after: #{doc[key]["address"]["country"]}"
        lat = doc[key]["lat"]
        lon = doc[key]["lon"]
        User.override({:_id => u.id},
                    {:position => {lat: lat, long: lon},
                      :address => doc[key]["address"]})
#         FIXME
#         Comment.override({:user_id => u.id},
#                     {:position => GeoPosition.new(lat, lon),
#                       :address => doc[key]["address"]})
        Question.override({:user_id => u.id},
                    {:position => {lat: lat, long: lon},
                      :address => doc[key]["address"]})
        Answer.override({:user_id => u.id},
                    {:position => {lat: lat, long: lon},
                      :address => doc[key]["address"]})
      end
    end
  end

  task :update_questions_widgets => [:environment] do
    c=Group.count
    i=1
    Group.all.each do |g|
      [SharingButtonsWidget.new, ModInfoWidget.new, QuestionBadgesWidget.new,
       QuestionStatsWidget.new, QuestionTagsWidget.new, RelatedQuestionsWidget.new,
       TagListWidget.new, CurrentTagsWidget.new].each do |w|
        g.widgets << w
      end
      g.save
      p "(#{i}/#{c}) Updated widgets for group #{g.name}"
      i+=1
    end
  end
end

{'fraction_non_alphanumeric': 0.14796039603960395, 'fraction_numerical': 0.0014257425742574257, 'mean_word_length': 2.600228115198175, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 111, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3058201', 'n_tokens_mistral': 4037, 'n_tokens_neox': 3962, 'n_words': 801}