<commit_before># coding: UTF-8
require_relative '../spec_helper'

describe CommonData do

  before(:each) do
    Typhoeus::Expectation.clear
    @common_data = CommonData.new('http://common-data.example.com/api/v1/viz?type=table&privacy=public')
    @common_data.stubs(:config).with('protocol', 'https').returns('https')
    @common_data.stubs(:config).with('username').returns('common-data')
    @common_data.stubs(:config).with('base_url').returns(nil)
    @common_data.stubs(:config).with('api_key').returns('wadus')
    @common_data.stubs(:config).with('format', 'shp').returns('shp')
    CommonDataRedisCache.new.invalidate
  end

  after(:all) do
    Typhoeus::Expectation.clear
  end

  it 'should return empty datasets response and notify error for SQL API error response' do
    stub_api_response(503)
    CartoDB.expects(:notify_error).with('common-data empty', { rows: [] , url: 'http://common-data.example.com/api/v1/viz?type=table&privacy=public'})

    @common_data.datasets.should eq []
  end

  it 'should return empty datasets and notify error for invalid json' do
    stub_api_response(200, INVALID_JSON_RESPONSE)
    CartoDB.expects(:notify_error).with('common-data empty', { rows: [] , url: 'http://common-data.example.com/api/v1/viz?type=table&privacy=public'})

    @common_data.datasets.should eq []
  end

  it 'should return correct datasets for default stub response' do
    stub_valid_api_response
    CartoDB.expects(:notify_error).times(0)

    @common_data.datasets.select{ |d| d["name"] =~ /meta_/}.length.should eq 0
    @common_data.datasets.length.should eq 6
  end

  it 'should use name if the display_name is null' do
    stub_valid_api_response
    CartoDB.expects(:notify_error).times(0)

    @common_data.datasets.first['display_name'].should eq @common_data.datasets.first['name']
  end

  it 'reads the attributions' do
    stub_valid_api_response
    @common_data.datasets.first['attributions'].should eq 'CartoDB Inc.'
  end

  it 'categories should be an array' do
    stub_valid_api_response
    CartoDB.expects(:notify_error).times(0)

    (@common_data.datasets.first['tags'].is_a? Array).should eq true
  end


  it 'should use SQL API V2 for export URLs' do
    stub_valid_api_response

    @common_data.datasets.first['url'].should match (/common-data\.localhost\.lan\/api\/v2/)
  end
<commit_msg>Added source test<commit_after># coding: UTF-8
require_relative '../spec_helper'

describe CommonData do

  before(:each) do
    Typhoeus::Expectation.clear
    @common_data = CommonData.new('http://common-data.example.com/api/v1/viz?type=table&privacy=public')
    @common_data.stubs(:config).with('protocol', 'https').returns('https')
    @common_data.stubs(:config).with('username').returns('common-data')
    @common_data.stubs(:config).with('base_url').returns(nil)
    @common_data.stubs(:config).with('api_key').returns('wadus')
    @common_data.stubs(:config).with('format', 'shp').returns('shp')
    CommonDataRedisCache.new.invalidate
  end

  after(:all) do
    Typhoeus::Expectation.clear
  end

  it 'should return empty datasets response and notify error for SQL API error response' do
    stub_api_response(503)
    CartoDB.expects(:notify_error).with('common-data empty', { rows: [] , url: 'http://common-data.example.com/api/v1/viz?type=table&privacy=public'})

    @common_data.datasets.should eq []
  end

  it 'should return empty datasets and notify error for invalid json' do
    stub_api_response(200, INVALID_JSON_RESPONSE)
    CartoDB.expects(:notify_error).with('common-data empty', { rows: [] , url: 'http://common-data.example.com/api/v1/viz?type=table&privacy=public'})

    @common_data.datasets.should eq []
  end

  it 'should return correct datasets for default stub response' do
    stub_valid_api_response
    CartoDB.expects(:notify_error).times(0)

    @common_data.datasets.select{ |d| d["name"] =~ /meta_/}.length.should eq 0
    @common_data.datasets.length.should eq 6
  end

  it 'should use name if the display_name is null' do
    stub_valid_api_response
    CartoDB.expects(:notify_error).times(0)

    @common_data.datasets.first['display_name'].should eq @common_data.datasets.first['name']
  end

  it 'reads the source and attributions' do
    stub_valid_api_response
    @common_data.datasets.first['source'].should eq '[Los Angeles opendata](http://datos.losangeles.com/)'
    @common_data.datasets.first['attributions'].should eq 'CartoDB Inc.'
  end

  it 'categories should be an array' do
    stub_valid_api_response
    CartoDB.expects(:notify_error).times(0)

    (@common_data.datasets.first['tags'].is_a? Array).should eq true
  end


  it 'should use SQL API V2 for export URLs' do
    stub_valid_api_response

    @common_data.datasets.first['url'].should match (/common-data\.localhost\.lan\/api\/v2/)
  end

{'fraction_non_alphanumeric': 0.13740141137401413, 'fraction_numerical': 0.007056870070568701, 'mean_word_length': 4.463718820861678, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6936772', 'n_tokens_mistral': 1654, 'n_tokens_neox': 1607, 'n_words': 309}