<commit_before>require 'spec_helper'

describe HeyDan::Base do

  let(:dir) {File.join('spec', 'tmp1')}

  it 'setups_folders' do
    HeyDan::folders = {
      jurisdictions: 'jurisdictions',
      sources: 'sources',
      downloads: 'downloads' 
    }

    expect(HeyDan::folders).to eq ({:jurisdictions=>"jurisdictions", :sources=>"sources", :downloads=>"downloads"})
    HeyDan::Base.setup_folders(dir)
    expect(HeyDan::folders).to eq ({:jurisdictions=>"#{dir}/jurisdictions", :sources=>"#{dir}/sources", :downloads=>"#{dir}/downloads"})
  end

  it 'setup_dir' do
    expect(Dir.exist? dir).to be false
    HeyDan::Base.setup_dir(dir)
    expect(Dir.exist? dir).to be true
  end

  it 'create_settings_file' do
    settings_file = File.join(dir, 'heydan_settings.yml')
    expect(File.exist?(settings_file)).to be false
    HeyDan::Base.create_settings_file(dir)
    expect(File.exist?(settings_file)).to be true
    expect(YAML.load(File.read(settings_file))).to eq ({:help=>true, :folders=>{:jurisdictions=>"spec/tmp1/spec/tmp/jurisdictions", :sources=>"spec/tmp1/spec/tmp/sources", :downloads=>"spec/tmp1/spec/tmp/downloads", :datasets=>"spec/tmp1/spec/tmp/datasets"}, :sources=>{:heydan_sources=>"<EMAIL>:danmelton/heydan_sources.git"}})
  end

  it 'load_settings_file' do
    HeyDan::Base.create_settings_file(dir)
    HeyDan::folders = {
      jurisdictions: 'jurisdictions',
      sources: 'sources',
      downloads: 'downloads' 
    }
    expect(HeyDan::folders).to eq ({:jurisdictions=>"jurisdictions", :sources=>"sources", :downloads=>"downloads"})
    HeyDan::Base.load_settings_file(File.join(dir, 'heydan_settings.yml'))
    expect(HeyDan::folders).to eq ({:jurisdictions=>"spec/tmp1/spec/tmp/jurisdictions", :sources=>"spec/tmp1/spec/tmp/sources", :downloads=>"spec/tmp1/spec/tmp/downloads", :datasets=>"spec/tmp1/spec/tmp/datasets"})
  end

  it 'create_folders' do
    HeyDan::Base.setup_folders(dir)
    HeyDan::folders.keys.each do |folder|
      expect(Dir.exist?(HeyDan::folders[folder])).to be false
    end
    HeyDan::Base.create_folders
    HeyDan::folders.keys.each do |folder|
      expect(Dir.exist?(HeyDan::folders[folder])).to be true
    end
  end<commit_msg>updateing base spec<commit_after>require 'spec_helper'

describe HeyDan::Base do

  let(:dir) {File.join('spec', 'tmp1')}

  it 'setups_folders' do
    HeyDan::folders = {
      jurisdictions: 'jurisdictions',
      sources: 'sources',
      downloads: 'downloads' 
    }

    expect(HeyDan::folders).to eq ({:jurisdictions=>"jurisdictions", :sources=>"sources", :downloads=>"downloads"})
    HeyDan::Base.setup_folders(dir)
    expect(HeyDan::folders).to eq ({:jurisdictions=>"#{dir}/jurisdictions", :sources=>"#{dir}/sources", :downloads=>"#{dir}/downloads"})
  end

  it 'setup_dir' do
    expect(Dir.exist? dir).to be false
    HeyDan::Base.setup_dir(dir)
    expect(Dir.exist? dir).to be true
  end

  it 'create_settings_file' do
    settings_file = File.join(dir, 'heydan_settings.yml')
    expect(File.exist?(settings_file)).to be false
    HeyDan::Base.create_settings_file(dir)
    expect(File.exist?(settings_file)).to be true
    expect(YAML.load(File.read(settings_file))).to eq ({:help=>true, :folders=>{:jurisdictions=>"spec/tmp1/spec/tmp/jurisdictions", :sources=>"spec/tmp1/spec/tmp/sources", :downloads=>"spec/tmp1/spec/tmp/downloads", :datasets=>"spec/tmp1/spec/tmp/datasets"}, :sources=>{:heydan_sources=>"https://github.com/danmelton/heydan_sources.git"}})
  end

  it 'load_settings_file' do
    HeyDan::Base.create_settings_file(dir)
    HeyDan::folders = {
      jurisdictions: 'jurisdictions',
      sources: 'sources',
      downloads: 'downloads' 
    }
    expect(HeyDan::folders).to eq ({:jurisdictions=>"jurisdictions", :sources=>"sources", :downloads=>"downloads"})
    HeyDan::Base.load_settings_file(File.join(dir, 'heydan_settings.yml'))
    expect(HeyDan::folders).to eq ({:jurisdictions=>"spec/tmp1/spec/tmp/jurisdictions", :sources=>"spec/tmp1/spec/tmp/sources", :downloads=>"spec/tmp1/spec/tmp/downloads", :datasets=>"spec/tmp1/spec/tmp/datasets"})
  end

  it 'create_folders' do
    HeyDan::Base.setup_folders(dir)
    HeyDan::folders.keys.each do |folder|
      expect(Dir.exist?(HeyDan::folders[folder])).to be false
    end
    HeyDan::Base.create_folders
    HeyDan::folders.keys.each do |folder|
      expect(Dir.exist?(HeyDan::folders[folder])).to be true
    end
  end
{'fraction_non_alphanumeric': 0.17403127124405166, 'fraction_numerical': 0.004078857919782461, 'mean_word_length': 4.94078061911171, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 46, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7175016', 'n_tokens_mistral': 1642, 'n_tokens_neox': 1571, 'n_words': 221}