<commit_before>
require 'cgi'
require 'tempfile'
require 'digest/md5'
require 'stringio'
require 'enumerator'
require 'base64'

require 'aws-sdk'
require 'nokogiri'
require 'httparty'
require 'image_size'
require 'parallel'
require 'm3u8'

require 'discordrb'
require 'yaml'
require 'json'

require 'modules/buddy-functionality'

module BuddyBot::Modules::Tistory
  extend Discordrb::EventContainer

  @@is_testbot = false

  @@s3 = nil
  @@s3_bucket = nil

  @@s3_bucket_name = nil

  @@pages = []
  @@pages_special = []
  @@pages_downloaded = {}
  @@sendanywhere_downloaded = {}
  @@twitter_list = []
  @@twitter_downloaded = {}

  @@initialized = false

  @@number_of_processes = 50

  @@abort_tistory_queue_in_progress = false
  @@abort_twitter_queue_in_progress = false

  @@twt_consumer_key = nil
  @@twt_consumer_secret = nil
  @@twt_app_bearer = nil

  def self.scan_bot_files()
    @@pages = YAML.load_file(BuddyBot.path("content/tistory-list.yml")) || []
    @@pages_special = YAML.load_file(BuddyBot.path("content/tistory-special-list.yml")) || []
    @@pages_downloaded = YAML.load_file(BuddyBot.path("content/tistory-pages-downloaded.yml")) || {}
    @@sendanywhere_downloaded = YAML.load_file(BuddyBot.path("content/downloaded-sendanywhere.yml")) || {}

    @@twitter_list = YAML.load_file(BuddyBot.path("content/pages-twitter.yml")) || []
    @@twitter_downloaded = YAML.load_file(BuddyBot.path("content/downloaded-twitter.yml")) || {}

    bot_config = YAML.load_file(BuddyBot.path("content/bot.yml")) || {}
    localconf = YAML::load(File.read(BuddyBot.localconf_filename))

    @@is_testbot = localconf["appid"] == bot_config["bot_test_id"]
  end

  def self.log(message, bot)
    BuddyBot::Modules::BuddyFunctionality.log message, bot, Struct.new(:id).new(123456)
  end

  def self.log_warning(message, bot)
    BuddyBot::Modules::BuddyFunctionality.log message, bot, Struct.new(:id).new(12345678)
  end

  ready do |event|
    if !@@initialized
      self.scan_bot_files()
      @@initialized = true
    end
    self.log ":information_desk_person: Ready to upload to '#{@@s3_bucket_name}'", event.bot

    @@twt_app_bearer = self.twitter_retrieve_app_bearer(@@twt_consumer_key, @@twt_consumer_secret, event.bot)

    # self.process_mobile_page("http://gfriendcom.tistory.com/m/145", "http://gfriendcom.tistory.com/m/145", "gfriendcom", "145", event, true)
    # self.process_tweet("https://twitter.com/Candle4_YB/status/1007759490588934144", event)
    # puts self.process_tweet("https://twitter.com/Mochi_Yellow/status/1007198387693748224", event)
    # puts self.process_tweet("https://twitter.com/mystarmyangel/status/1007664325916438528", event)
    # self.process_tweet("https://twitter.com/gagadoli/status/1007639571150991361", event)
    # self.process_tweet("https://twitter.com/_Simplykpop/status/994782508083539968", event)
  end

  def self.set_s3_bucket_name(name)
    @@s3_bucket_name = name
    @@s3 = Aws::S3::Resource.new()
    @@s3_bucket = @@s3.bucket(@@s3_bucket_name)
  end

  def self.set_twitter_credentials(key, secret)
    @@twt_consumer_key = key
    @@twt_consumer_secret = secret
  end

  # invoke this command if you want to e.g. add new audio clips or memes, but don't want to restart the bot. for now, you also have to invoke e.g. #audio-load manually afterwards.
  message(content: "!crawler-git-sync") do |event|
    next unless event.user.id == 139342974776639489
    event.channel.split_send "Done.\n#{`cd #{BuddyBot.path} && git pull && git add content/tistory-list.yml content/tistory-pages-downloaded.yml content/pages-twitter.yml content/downloaded-twitter.yml content/downloaded-sendanywhere.yml && git commit -m "tistory: update pages" && git push`}"
  end

  message(start_with: /^!tistory\s/i) do |event|
    next if event.user.bot_account?
    data = event.content.scan(/^!tistory\s+<?(.*?)\s*>?\s*$/i)[0]
    if !data
      event.send_message ":warning: You need to specify a tistory page link... #{BuddyBot::Modules::BuddyFunctionality.random_derp_emoji()}"
      next
    end

    orig_input = url = data[0].downcase

    if url !~ /https?:\/\/.*?\.tistory\.com(\/m)?\/\d+$/
      event.send_message ":warning: URL is not a specific page, try e.g. <http://gfriendcom.tistory.com/163>"
      next
    end
<commit_msg>disable sanity check for now, not really used<commit_after>
require 'cgi'
require 'tempfile'
require 'digest/md5'
require 'stringio'
require 'enumerator'
require 'base64'

require 'aws-sdk'
require 'nokogiri'
require 'httparty'
require 'image_size'
require 'parallel'
require 'm3u8'

require 'discordrb'
require 'yaml'
require 'json'

require 'modules/buddy-functionality'

module BuddyBot::Modules::Tistory
  extend Discordrb::EventContainer

  @@is_testbot = false

  @@s3 = nil
  @@s3_bucket = nil

  @@s3_bucket_name = nil

  @@pages = []
  @@pages_special = []
  @@pages_downloaded = {}
  @@sendanywhere_downloaded = {}
  @@twitter_list = []
  @@twitter_downloaded = {}

  @@initialized = false

  @@number_of_processes = 50

  @@abort_tistory_queue_in_progress = false
  @@abort_twitter_queue_in_progress = false

  @@twt_consumer_key = nil
  @@twt_consumer_secret = nil
  @@twt_app_bearer = nil

  def self.scan_bot_files()
    @@pages = YAML.load_file(BuddyBot.path("content/tistory-list.yml")) || []
    @@pages_special = YAML.load_file(BuddyBot.path("content/tistory-special-list.yml")) || []
    @@pages_downloaded = YAML.load_file(BuddyBot.path("content/tistory-pages-downloaded.yml")) || {}
    @@sendanywhere_downloaded = YAML.load_file(BuddyBot.path("content/downloaded-sendanywhere.yml")) || {}

    @@twitter_list = YAML.load_file(BuddyBot.path("content/pages-twitter.yml")) || []
    @@twitter_downloaded = YAML.load_file(BuddyBot.path("content/downloaded-twitter.yml")) || {}

    bot_config = YAML.load_file(BuddyBot.path("content/bot.yml")) || {}
    localconf = YAML::load(File.read(BuddyBot.localconf_filename))

    @@is_testbot = localconf["appid"] == bot_config["bot_test_id"]
  end

  def self.log(message, bot)
    BuddyBot::Modules::BuddyFunctionality.log message, bot, Struct.new(:id).new(123456)
  end

  def self.log_warning(message, bot)
    BuddyBot::Modules::BuddyFunctionality.log message, bot, Struct.new(:id).new(12345678)
  end

  ready do |event|
    if !@@initialized
      self.scan_bot_files()
      @@initialized = true
    end
    self.log ":information_desk_person: Ready to upload to '#{@@s3_bucket_name}'", event.bot

    @@twt_app_bearer = self.twitter_retrieve_app_bearer(@@twt_consumer_key, @@twt_consumer_secret, event.bot)

    # self.process_mobile_page("http://gfriendcom.tistory.com/m/145", "http://gfriendcom.tistory.com/m/145", "gfriendcom", "145", event, true)
    # self.process_tweet("https://twitter.com/Candle4_YB/status/1007759490588934144", event)
    # puts self.process_tweet("https://twitter.com/Mochi_Yellow/status/1007198387693748224", event)
    # puts self.process_tweet("https://twitter.com/mystarmyangel/status/1007664325916438528", event)
    # self.process_tweet("https://twitter.com/gagadoli/status/1007639571150991361", event)
    # self.process_tweet("https://twitter.com/_Simplykpop/status/994782508083539968", event)
  end

  def self.set_s3_bucket_name(name)
    @@s3_bucket_name = name
    @@s3 = Aws::S3::Resource.new()
    @@s3_bucket = @@s3.bucket(@@s3_bucket_name)
  end

  def self.set_twitter_credentials(key, secret)
    @@twt_consumer_key = key
    @@twt_consumer_secret = secret
  end

  # invoke this command if you want to e.g. add new audio clips or memes, but don't want to restart the bot. for now, you also have to invoke e.g. #audio-load manually afterwards.
  message(content: "!crawler-git-sync") do |event|
    next unless event.user.id == 139342974776639489
    event.channel.split_send "Done.\n#{`cd #{BuddyBot.path} && git pull && git add content/tistory-list.yml content/tistory-pages-downloaded.yml content/pages-twitter.yml content/downloaded-twitter.yml content/downloaded-sendanywhere.yml && git commit -m "tistory: update pages" && git push`}"
  end

  message(start_with: /^!tistory\s/i) do |event|
    next if event.user.bot_account?
    data = event.content.scan(/^!tistory\s+<?(.*?)\s*>?\s*$/i)[0]
    if !data
      event.send_message ":warning: You need to specify a tistory page link... #{BuddyBot::Modules::BuddyFunctionality.random_derp_emoji()}"
      next
    end

    orig_input = url = data[0].downcase

    # if url !~ /https?:\/\/.*?\.tistory\.com(\/m)?\/\d+$/
    #   event.send_message ":warning: URL is not a specific page, try e.g. <http://gfriendcom.tistory.com/163>"
    #   next
    # end

{'fraction_non_alphanumeric': 0.150511390051139, 'fraction_numerical': 0.03695955369595537, 'mean_word_length': 4.411949685534591, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '94516', 'n_tokens_mistral': 3450, 'n_tokens_neox': 3050, 'n_words': 535}