<commit_before>require 'rails_helper'
require "#{Rails.root}/lib/importers/category_importer"

describe CategoryImporter do
  let(:category) { 'Category:"Crocodile" Dundee' }
  let(:article_in_cat) { 'Michael_"Crocodile"_Dundee' }
  let(:article_in_subcat) { 'Crocodile_Dundee_in_Los_Angeles' }
  let(:wiki) { Wiki.default_wiki }

  it 'imports data for articles in a category' do
    VCR.use_cassette 'category_importer/import' do
      CategoryImporter.new(wiki)
                      .import_category(category)
      expect(Article.exists?(title: article_in_cat))
        .to be true # depth 0
      expect(Article.exists?(title: article_in_subcat))
        .to be false # depth 1
    end
  end

  it 'imports subcategories recursively' do
    VCR.use_cassette 'category_importer/import' do
      CategoryImporter.new(wiki, depth: 1)
                      .import_category(category)
      expect(Article.exists?(title: article_in_cat))
        .to be true # depth 0
      expect(Article.exists?(title: article_in_subcat))
        .to be true # depth 1
    end
  end

  it 'outputs filtered data about a category' do
    VCR.use_cassette 'category_importer/import' do
      CategoryImporter.new(wiki, depth: 1)
                      .import_category(category)
    end
    VCR.use_cassette 'category_importer/report_on_category' do
      output = CategoryImporter.new(wiki)
                               .report_on_category(category)
      expect(output).to include article_in_cat
    end
  end

  it 'imports missing data for category' do<commit_msg>Mark a flaky spec as pending<commit_after>require 'rails_helper'
require "#{Rails.root}/lib/importers/category_importer"

describe CategoryImporter do
  let(:category) { 'Category:"Crocodile" Dundee' }
  let(:article_in_cat) { 'Michael_"Crocodile"_Dundee' }
  let(:article_in_subcat) { 'Crocodile_Dundee_in_Los_Angeles' }
  let(:wiki) { Wiki.default_wiki }

  it 'imports data for articles in a category' do
    VCR.use_cassette 'category_importer/import' do
      CategoryImporter.new(wiki)
                      .import_category(category)
      expect(Article.exists?(title: article_in_cat))
        .to be true # depth 0
      expect(Article.exists?(title: article_in_subcat))
        .to be false # depth 1
    end
  end

  it 'imports subcategories recursively' do
    VCR.use_cassette 'category_importer/import' do
      pending 'This sometimes fails on travis for database timeout reasons.'

      CategoryImporter.new(wiki, depth: 1)
                      .import_category(category)
      expect(Article.exists?(title: article_in_cat))
        .to be true # depth 0
      expect(Article.exists?(title: article_in_subcat))
        .to be true # depth 1

      puts 'PASSED'
      raise 'this test passed — this time'
    end
  end

  it 'outputs filtered data about a category' do
    VCR.use_cassette 'category_importer/import' do
      CategoryImporter.new(wiki, depth: 1)
                      .import_category(category)
    end
    VCR.use_cassette 'category_importer/report_on_category' do
      output = CategoryImporter.new(wiki)
                               .report_on_category(category)
      expect(output).to include article_in_cat
    end
  end

  it 'imports missing data for category' do
{'fraction_non_alphanumeric': 0.08703646950658903, 'fraction_numerical': 0.003677597303095311, 'mean_word_length': 2.932530120481928, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3216039', 'n_tokens_mistral': 1055, 'n_tokens_neox': 1010, 'n_words': 239}