<commit_before>class PostScraper < Object
  SANDBOX_ID = Rails.env.production? ? 3 : 5
  BASE_ACCOUNTS = {
    'alicornucopia' => 'Alicorn',
    'pythbox' => 'Kappa',
    'lintamande' => 'lintamande',
    'marrinikari' => 'Marri',
    'peterxy' => 'Pedro',
    'peterverse' => 'Pedro',
    'curiousdiscoverer' => 'CuriousDiscoverer',
    'aestrix' => 'Aestrix',
    'unbitwise' => 'Unbitwise',
    'erinflight' => 'ErinFlight',
    'andaisq' => 'andaisq',
    'rockeye-stonetoe' => 'Rockeye',
    'rockeye_stonetoe' => 'Rockeye',
    'maggie-of-the-owls' => 'MaggieoftheOwls',
    'maggie_of_the_owls' => 'MaggieoftheOwls', # have both - and _ versions cause Dreamwidth supports both
    'nemoconsequentiae' => 'Nemo',
    'armokgob' => 'Armok',
  }

  attr_accessor :url, :post, :html_doc

  def initialize(url, board_id=nil, section_id=nil, status=nil, threaded_import=false, console_import=false, subject=nil)
    @board_id = board_id || SANDBOX_ID
    @section_id = section_id
    @status = status || Post::STATUS_COMPLETE
    url += (url.include?('?') ? '&' : '?') + 'style=site' unless url.include?('style=site')
    url += '&view=flat' unless url.include?('view=flat') || threaded_import
    @url = url
    @console_import = console_import<commit_msg>Add Timepoof as an importable base account<commit_after>class PostScraper < Object
  SANDBOX_ID = Rails.env.production? ? 3 : 5
  BASE_ACCOUNTS = {
    'alicornucopia' => 'Alicorn',
    'pythbox' => 'Kappa',
    'lintamande' => 'lintamande',
    'marrinikari' => 'Marri',
    'peterxy' => 'Pedro',
    'peterverse' => 'Pedro',
    'curiousdiscoverer' => 'CuriousDiscoverer',
    'aestrix' => 'Aestrix',
    'unbitwise' => 'Unbitwise',
    'erinflight' => 'ErinFlight',
    'andaisq' => 'andaisq',
    'rockeye-stonetoe' => 'Rockeye',
    'rockeye_stonetoe' => 'Rockeye',
    'maggie-of-the-owls' => 'MaggieoftheOwls',
    'maggie_of_the_owls' => 'MaggieoftheOwls', # have both - and _ versions cause Dreamwidth supports both
    'nemoconsequentiae' => 'Nemo',
    'armokgob' => 'Armok',
    'timepoof' => 'Timepoof',
  }

  attr_accessor :url, :post, :html_doc

  def initialize(url, board_id=nil, section_id=nil, status=nil, threaded_import=false, console_import=false, subject=nil)
    @board_id = board_id || SANDBOX_ID
    @section_id = section_id
    @status = status || Post::STATUS_COMPLETE
    url += (url.include?('?') ? '&' : '?') + 'style=site' unless url.include?('style=site')
    url += '&view=flat' unless url.include?('view=flat') || threaded_import
    @url = url
    @console_import = console_import
{'fraction_non_alphanumeric': 0.1720388349514563, 'fraction_numerical': 0.0015533980582524273, 'mean_word_length': 4.152, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4011377', 'n_tokens_mistral': 1031, 'n_tokens_neox': 986, 'n_words': 175}