<commit_before>#!/usr/bin/env ruby
# -*- coding: utf-8 -*-

require 'libxml'
require 'htmlentities'
require 'pg'

require File.join(File.dirname(__FILE__), "..", "config", "boot")
require File.join(File.dirname(__FILE__), "..", "config", "environment")

ActiveRecord::Base.record_timestamps = false
Rails.logger = Logger.new('/dev/null')
ActiveRecord::Base.logger = Rails.logger

$old_db = PG.connect(dbname: 'oldusers')

$default_forum = CfForum.where(slug: 'self').first!
meta = CfForum.where(slug: 'meta').first!
$map = {
  # meta forums
  'ZU DIESEM FORUM' => meta,
  'SELFHTML' => meta,
  'SELFHTML-WIKI' => meta
}

$ids = {}
$coder = HTMLEntities.new

def check_null_bytes(msg)
  msg.content.gsub!(/\0/, '')
  msg.subject.gsub!(/\0/, '')
  msg.author.gsub!(/\0/, '')
  msg.email.gsub!(/\0/, '') unless msg.email.blank?
  msg.homepage.gsub!(/\0/, '') unless msg.homepage.blank?
end

def handle_messages(old_msg, x_msg, thread)
  mid = x_msg['id'].gsub(/^m/, '')

  msg = CfMessage.
        where(mid: mid,
              thread_id: thread.thread_id).
        first

  if msg.blank?
    $stderr.puts "NEW MESSAGE!"
    exit

    the_date = Time.at(x_msg.find_first('./Header/Date')['longSec'].force_encoding('utf-8').to_i)
    the_date = DateTime.parse("1970-01-01 00:00:00").to_time if the_date.blank?

    msg = CfMessage.new(
      mid: mid,
      author: x_msg.find_first('./Header/Author/Name').content.force_encoding('utf-8'),
      subject: x_msg.find_first('./Header/Subject').content.force_encoding('utf-8'),

      upvotes: x_msg['votingGood'].to_i,
      downvotes: x_msg['votingBad'].to_i,

      deleted: x_msg['invisible'] == '1',

      created_at: the_date,
      updated_at: the_date,

      content: x_msg.find_first('./MessageContent').content.force_encoding('utf-8'),
      parent_id: old_msg ? old_msg.message_id : nil,
      thread_id: thread.thread_id,
      forum_id: thread.forum_id
    )

    # cat      = x_msg.find_first('./Header/Category').content.force_encoding('utf-8')
    email    = x_msg.find_first('./Header/Author/Email').content.force_encoding('utf-8')
    homepage = x_msg.find_first('./Header/Author/HomepageUrl').content.force_encoding('utf-8')

    # msg.category        = cat      unless cat.empty?
    msg.email    = email    unless email.blank?
    msg.homepage = homepage unless homepage.blank?

    check_null_bytes(msg)

    msg.save(validate: false)

    category = x_msg.find_first("./Header/Category").content.force_encoding('utf-8')

    if not category.blank?
      category = category.downcase.strip

      t = CfTag.find_by_forum_id_and_tag_name thread.forum_id, category

      begin
        t = CfTag.create!(:tag_name => category, forum_id: thread.forum_id) if t.blank?
      rescue ActiveRecord::RecordNotUnique
        t = CfTag.find_by_forum_id_and_tag_name! thread.forum_id, category
      end

      CfMessageTag.create!(
        tag_id: t.tag_id,
        message_id: msg.message_id
      )
    end

    x_msg.find('./Header/Flags/Flag').each do |f|
      if f['name'] == 'UserName' then
        uname = f.content.force_encoding('utf-8')

        usr = CfUser.find_by_username(uname)
        if !usr then
          email = nil
          $old_db.exec("SELECT email FROM auth WHERE username = '" + uname + "'") do |result|
            result.each do |row|
              email = row.values_at('email').first
            end
          end

          usr = CfUser.new(username: uname, created_at: the_date, updated_at: the_date, email: email)
          usr.skip_confirmation!

          begin
            usr.save!(validate: false)
          rescue ActiveRecord::RecordNotUnique
            usr = CfUser.where(username: uname).first

            if usr.blank?
              usr.email = nil
              usr.save!(validate: false)
            end
          end
        end

        msg.user_id = usr.id
        msg.save
      else
        msg.flags[f['name']] = f.content.force_encoding('utf-8')
      end
    end

  else
    msg.content = x_msg.find_first('./MessageContent').content.force_encoding('utf-8')
    msg.format = 'cforum'
    msg.save
  end

  x_msg.find('./Message').each do |m|
    handle_messages(msg, m, thread)
  end

  msg
end

def handle_doc(doc, opts = {})
  x_thread = doc.find_first('/Forum/Thread')
  the_date = Time.at(x_thread.find_first('./Message/Header/Date')['longSec'].force_encoding('utf-8').to_i)
  the_date = DateTime.parse("1970-01-01 00:00:00").to_time if the_date.blank?

  thread = CfThread.
           where(tid: x_thread['id'].force_encoding('utf-8')[1..-1]).
           where("EXTRACT('year' FROM created_at) = ?", the_date.year).
           first

  if thread.blank?
    $stderr.puts "NEW THREAD!"
    exit
    forum_name = x_thread.find_first("./Message/Header/Category").content.force_encoding('utf-8')

    subject = x_thread.find_first('./Message/Header/Subject').content.force_encoding('utf-8')

    forum = $map[forum_name] || $default_forum

    thread = CfThread.new(<commit_msg>fix: use utc time when getting the year<commit_after>#!/usr/bin/env ruby
# -*- coding: utf-8 -*-

require 'libxml'
require 'htmlentities'
require 'pg'

require File.join(File.dirname(__FILE__), "..", "config", "boot")
require File.join(File.dirname(__FILE__), "..", "config", "environment")

ActiveRecord::Base.record_timestamps = false
Rails.logger = Logger.new('/dev/null')
ActiveRecord::Base.logger = Rails.logger

$old_db = PG.connect(dbname: 'oldusers')

$default_forum = CfForum.where(slug: 'self').first!
meta = CfForum.where(slug: 'meta').first!
$map = {
  # meta forums
  'ZU DIESEM FORUM' => meta,
  'SELFHTML' => meta,
  'SELFHTML-WIKI' => meta
}

$ids = {}
$coder = HTMLEntities.new

def check_null_bytes(msg)
  msg.content.gsub!(/\0/, '')
  msg.subject.gsub!(/\0/, '')
  msg.author.gsub!(/\0/, '')
  msg.email.gsub!(/\0/, '') unless msg.email.blank?
  msg.homepage.gsub!(/\0/, '') unless msg.homepage.blank?
end

def handle_messages(old_msg, x_msg, thread)
  mid = x_msg['id'].gsub(/^m/, '')

  msg = CfMessage.
        where(mid: mid,
              thread_id: thread.thread_id).
        first

  if msg.blank?
    $stderr.puts "NEW MESSAGE!"
    exit

    the_date = Time.at(x_msg.find_first('./Header/Date')['longSec'].force_encoding('utf-8').to_i)
    the_date = DateTime.parse("1970-01-01 00:00:00").to_time if the_date.blank?

    msg = CfMessage.new(
      mid: mid,
      author: x_msg.find_first('./Header/Author/Name').content.force_encoding('utf-8'),
      subject: x_msg.find_first('./Header/Subject').content.force_encoding('utf-8'),

      upvotes: x_msg['votingGood'].to_i,
      downvotes: x_msg['votingBad'].to_i,

      deleted: x_msg['invisible'] == '1',

      created_at: the_date,
      updated_at: the_date,

      content: x_msg.find_first('./MessageContent').content.force_encoding('utf-8'),
      parent_id: old_msg ? old_msg.message_id : nil,
      thread_id: thread.thread_id,
      forum_id: thread.forum_id
    )

    # cat      = x_msg.find_first('./Header/Category').content.force_encoding('utf-8')
    email    = x_msg.find_first('./Header/Author/Email').content.force_encoding('utf-8')
    homepage = x_msg.find_first('./Header/Author/HomepageUrl').content.force_encoding('utf-8')

    # msg.category        = cat      unless cat.empty?
    msg.email    = email    unless email.blank?
    msg.homepage = homepage unless homepage.blank?

    check_null_bytes(msg)

    msg.save(validate: false)

    category = x_msg.find_first("./Header/Category").content.force_encoding('utf-8')

    if not category.blank?
      category = category.downcase.strip

      t = CfTag.find_by_forum_id_and_tag_name thread.forum_id, category

      begin
        t = CfTag.create!(:tag_name => category, forum_id: thread.forum_id) if t.blank?
      rescue ActiveRecord::RecordNotUnique
        t = CfTag.find_by_forum_id_and_tag_name! thread.forum_id, category
      end

      CfMessageTag.create!(
        tag_id: t.tag_id,
        message_id: msg.message_id
      )
    end

    x_msg.find('./Header/Flags/Flag').each do |f|
      if f['name'] == 'UserName' then
        uname = f.content.force_encoding('utf-8')

        usr = CfUser.find_by_username(uname)
        if !usr then
          email = nil
          $old_db.exec("SELECT email FROM auth WHERE username = '" + uname + "'") do |result|
            result.each do |row|
              email = row.values_at('email').first
            end
          end

          usr = CfUser.new(username: uname, created_at: the_date, updated_at: the_date, email: email)
          usr.skip_confirmation!

          begin
            usr.save!(validate: false)
          rescue ActiveRecord::RecordNotUnique
            usr = CfUser.where(username: uname).first

            if usr.blank?
              usr.email = nil
              usr.save!(validate: false)
            end
          end
        end

        msg.user_id = usr.id
        msg.save
      else
        msg.flags[f['name']] = f.content.force_encoding('utf-8')
      end
    end

  else
    msg.content = x_msg.find_first('./MessageContent').content.force_encoding('utf-8')
    msg.format = 'cforum'
    msg.save
  end

  x_msg.find('./Message').each do |m|
    handle_messages(msg, m, thread)
  end

  msg
end

def handle_doc(doc, opts = {})
  x_thread = doc.find_first('/Forum/Thread')
  the_date = Time.at(x_thread.find_first('./Message/Header/Date')['longSec'].force_encoding('utf-8').to_i)
  the_date = DateTime.parse("1970-01-01 00:00:00").to_time if the_date.blank?

  thread = CfThread.
           where(tid: x_thread['id'].force_encoding('utf-8')[1..-1]).
           where("EXTRACT('year' FROM created_at) = ?", the_date.utc.year).
           first

  if thread.blank?
    $stderr.puts "NEW THREAD!"
    exit
    forum_name = x_thread.find_first("./Message/Header/Category").content.force_encoding('utf-8')

    subject = x_thread.find_first('./Message/Header/Subject').content.force_encoding('utf-8')

    forum = $map[forum_name] || $default_forum

    thread = CfThread.new(
{'fraction_non_alphanumeric': 0.15304695304695304, 'fraction_numerical': 0.01038961038961039, 'mean_word_length': 2.7992409867172676, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '942424', 'n_tokens_mistral': 3555, 'n_tokens_neox': 3395, 'n_words': 578}