<commit_before># encoding: utf-8
require "ryodo/suffix_list_fetcher"
require File.expand_path("../../spec_helper.rb", __FILE__)

describe Ryodo::SuffixListFetcher do

  before do
    @custom_uri = "http://custom.suffix.list.example.com/foo-bar.dat"
    @custom_storage = "#{RYODO_TMP_ROOT}/custom-storage-path.dat"
  end

  after do
    Dir["#{RYODO_TMP_ROOT}/**/*"].each do |file|
      File.delete(file)
    end
  end

  it "#new creates a new instance with predefined vars" do
    fetcher = described_class.new

    fetcher.instance_variable_get("@uri").should   == URI(Ryodo::PUBLIC_SUFFIX_DATA_URI)
    fetcher.instance_variable_get("@store").should == Ryodo::PUBLIC_SUFFIX_STORE
  end

  it "#new creates a new instance with custom vars" do

    fetcher = described_class.new(@custom_uri, @custom_storage)

    fetcher.instance_variable_get("@uri").should   == URI(@custom_uri)
    fetcher.instance_variable_get("@store").should == @custom_storage
  end

  context "data retrieval and storage" do

    let(:fetcher){ described_class.new }

    it "#fetch_data retrieves remote data" do
      fetcher.fetch_data

      fetcher.instance_variable_get("@fetched_data").first.should =~ /BEGIN LICENSE BLOCK/
    end

    it "#save_data stores fetched data into file (as cleaned set)" do
      fetcher.fetch_data
      fetcher.instance_variable_set("@prepared_data", ["dummy data"])

      File.should_receive(:open).with(Ryodo::PUBLIC_SUFFIX_STORE,"w")
      fetcher.save_data
    end

    it "#prepare_data manipulates and cleans data for storage" do
      fetcher.fetch_data
      prepared_data = fetcher.prepare_data

      prepared_data.should be_an(Array)
      prepared_data.none?{|e| e =~ /^\/\//}.should be_true # no comment lines
      prepared_data.none?{|e| e =~ /^\n/}.should   be_true # no empty lines<commit_msg>Fix spec<commit_after># encoding: utf-8
require "ryodo/suffix_list_fetcher"
require File.expand_path("../../spec_helper.rb", __FILE__)

describe Ryodo::SuffixListFetcher do

  before do
    @custom_uri = "http://custom.suffix.list.example.com/foo-bar.dat"
    @custom_storage = "#{RYODO_TMP_ROOT}/custom-storage-path.dat"
  end

  after do
    Dir["#{RYODO_TMP_ROOT}/**/*"].each do |file|
      File.delete(file)
    end
  end

  it "#new creates a new instance with predefined vars" do
    fetcher = described_class.new

    fetcher.instance_variable_get("@uri").should   == URI(Ryodo::PUBLIC_SUFFIX_DATA_URI)
    fetcher.instance_variable_get("@store").should == Ryodo::PUBLIC_SUFFIX_STORE
  end

  it "#new creates a new instance with custom vars" do

    fetcher = described_class.new(@custom_uri, @custom_storage)

    fetcher.instance_variable_get("@uri").should   == URI(@custom_uri)
    fetcher.instance_variable_get("@store").should == @custom_storage
  end

  context "data retrieval and storage" do

    let(:fetcher){ described_class.new }

    it "#fetch_data retrieves remote data" do
      fetcher.fetch_data

      fetcher.instance_variable_get("@fetched_data").first.should =~ /This Source Code Form is subject to the terms of the Mozilla Public/
    end

    it "#save_data stores fetched data into file (as cleaned set)" do
      fetcher.fetch_data
      fetcher.instance_variable_set("@prepared_data", ["dummy data"])

      File.should_receive(:open).with(Ryodo::PUBLIC_SUFFIX_STORE,"w")
      fetcher.save_data
    end

    it "#prepare_data manipulates and cleans data for storage" do
      fetcher.fetch_data
      prepared_data = fetcher.prepare_data

      prepared_data.should be_an(Array)
      prepared_data.none?{|e| e =~ /^\/\//}.should be_true # no comment lines
      prepared_data.none?{|e| e =~ /^\n/}.should   be_true # no empty lines
{'fraction_non_alphanumeric': 0.12009737625101434, 'fraction_numerical': 0.0005409791723018664, 'mean_word_length': 3.765463917525773, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3209653', 'n_tokens_mistral': 1247, 'n_tokens_neox': 1193, 'n_words': 264}