<commit_before>require File.dirname(__FILE__) + '/../../spec_helper'
require File.dirname(__FILE__) + '/data_store_spec'

describe Dragonfly::DataStorage::FileDataStore do
  
  def touch_file(filename)
    FileUtils.mkdir_p(File.dirname(filename))
    FileUtils.touch(filename)
  end
  
  before(:each) do
    @data_store = Dragonfly::DataStorage::FileDataStore.new
    @data_store.root_path = '/var/tmp/dragonfly_test'
  end
  
  after(:each) do
    # Clean up created files
    FileUtils.rm_rf("#{@data_store.root_path}")
  end
  
  it_should_behave_like 'data_store'
  
  before(:each) do
    @temp_object = Dragonfly::TempObject.new('goobydoo')
  end
  
  describe "store" do
    
    def it_should_write_to_file(storage_path, temp_object)
      temp_object.should_receive(:to_file).with(storage_path).and_return(mock('file', :close => nil))
    end
    
    before(:each) do
      # Set 'now' to a date in the past
      Time.stub!(:now).and_return Time.mktime(1984,"may",4,14,28,1)
      @file_pattern_prefix_without_root = '1984/05/04/'
      @file_pattern_prefix = "#{@data_store.root_path}/#{@file_pattern_prefix_without_root}"
    end
    
    it "should store the file in a folder based on date, with default filename" do
      it_should_write_to_file("#{@file_pattern_prefix}file", @temp_object)
      @data_store.store(@temp_object)
    end

    it "should use the temp_object name if it exists" do
      @temp_object.name = 'hello.there'
      it_should_write_to_file("#{@file_pattern_prefix}hello.there", @temp_object)
      @data_store.store(@temp_object)
    end

    it "should use the default file suffix if the temp_object name is blank" do
      @temp_object.name = ''
      it_should_write_to_file("#{@file_pattern_prefix}file", @temp_object)
      @data_store.store(@temp_object)
    end

    describe "when the filename already exists" do

      it "should use a different filename" do
        touch_file("#{@file_pattern_prefix}file")
        @data_store.should_receive(:disambiguate).with('file').and_return('file_2')
        it_should_write_to_file("#{@file_pattern_prefix}file_2", @temp_object)
        @data_store.store(@temp_object)
      end
    
      it "should use a different filename taking into account the name and ext" do
        @temp_object.name = 'hello.png'
        touch_file("#{@file_pattern_prefix}hello.png")
        @data_store.should_receive(:disambiguate).with('hello.png').and_return('blah.png')
        @data_store.store(@temp_object)
      end

      it "should keep trying until it finds a free filename" do
        touch_file("#{@file_pattern_prefix}file")
        touch_file("#{@file_pattern_prefix}file_2")
        @data_store.should_receive(:disambiguate).with('file').and_return('file_2')
        @data_store.should_receive(:disambiguate).with('file_2').and_return('file_3')
        it_should_write_to_file("#{@file_pattern_prefix}file_3", @temp_object)
        @data_store.store(@temp_object)
      end

    end

    describe "return value" do

      it "should return the filepath without the root of the stored file when a file name is not provided" do
        @data_store.store(@temp_object).should == "#{@file_pattern_prefix_without_root}file"
      end
    
      it "should return the filepath without the root of the stored file when a file name is provided" do
        @temp_object.name = 'hello.you.png'
        @data_store.store(@temp_object).should == "#{@file_pattern_prefix_without_root}hello.you.png"
      end
    
    end

  end
  
  describe "disambiguate" do
    it "should add a suffix" do
      @data_store.disambiguate('file').should =~ /^file_\w+$/
    end
    it "should add a suffix to the basename" do
      @data_store.disambiguate('file.png').should =~ /^file_\w+\.png$/
    end
    it "should be random(-ish)" do
      @data_store.disambiguate('file').should_not == @data_store.disambiguate('file')
    end
  end
  
  describe "errors" do

    it "should raise an error if it can't create a directory" do
      FileUtils.should_receive(:mkdir_p).and_raise(Errno::EACCES)
      lambda{ @data_store.store(@temp_object) }.should raise_error(Dragonfly::DataStorage::UnableToStore)
    end
  
    it "should raise an error if it can't create a file" do
      @temp_object.should_receive(:to_file).and_raise(Errno::EACCES)
      lambda{ @data_store.store(@temp_object) }.should raise_error(Dragonfly::DataStorage::UnableToStore)
    end

  end
  
  describe "retrieve" do
    it "should be able to retrieve any file, stored or not (and without extra data)" do
      FileUtils.mkdir_p("#{@data_store.root_path}/jelly_beans/are")
      File.open("#{@data_store.root_path}/jelly_beans/are/good", 'w'){|f| f.write('hey dog') }
      file, meta = @data_store.retrieve("jelly_beans/are/good")
      file.read.should == 'hey dog'
      meta.should be_nil
    end
  end
  
  describe "destroying" do

    it "should raise an error if the data doesn't exist" do
      lambda{
        @data_store.destroy('gooble/gubbub')
      }.should raise_error(Dragonfly::DataStorage::DataNotFound)
    end

    it "should prune empty directories when destroying" do
      uid = @data_store.store(@temp_object)
      @data_store.destroy(uid)<commit_msg>Fixed incorrect filedatastore spec<commit_after>require File.dirname(__FILE__) + '/../../spec_helper'
require File.dirname(__FILE__) + '/data_store_spec'

describe Dragonfly::DataStorage::FileDataStore do
  
  def touch_file(filename)
    FileUtils.mkdir_p(File.dirname(filename))
    FileUtils.touch(filename)
  end
  
  before(:each) do
    @data_store = Dragonfly::DataStorage::FileDataStore.new
    @data_store.root_path = '/var/tmp/dragonfly_test'
  end
  
  after(:each) do
    # Clean up created files
    FileUtils.rm_rf("#{@data_store.root_path}")
  end
  
  it_should_behave_like 'data_store'
  
  before(:each) do
    @temp_object = Dragonfly::TempObject.new('goobydoo')
  end
  
  describe "store" do
    
    def it_should_write_to_file(storage_path, temp_object)
      temp_object.should_receive(:to_file).with(storage_path).and_return(mock('file', :close => nil))
    end
    
    before(:each) do
      # Set 'now' to a date in the past
      Time.stub!(:now).and_return Time.mktime(1984,"may",4,14,28,1)
      @file_pattern_prefix_without_root = '1984/05/04/'
      @file_pattern_prefix = "#{@data_store.root_path}/#{@file_pattern_prefix_without_root}"
    end
    
    it "should store the file in a folder based on date, with default filename" do
      it_should_write_to_file("#{@file_pattern_prefix}file", @temp_object)
      @data_store.store(@temp_object)
    end

    it "should use the temp_object name if it exists" do
      @temp_object.name = 'hello.there'
      it_should_write_to_file("#{@file_pattern_prefix}hello.there", @temp_object)
      @data_store.store(@temp_object)
    end

    it "should use the default file suffix if the temp_object name is blank" do
      @temp_object.name = ''
      it_should_write_to_file("#{@file_pattern_prefix}file", @temp_object)
      @data_store.store(@temp_object)
    end

    describe "when the filename already exists" do

      it "should use a different filename" do
        touch_file("#{@file_pattern_prefix}file")
        @data_store.should_receive(:disambiguate).with('file').and_return('file_2')
        it_should_write_to_file("#{@file_pattern_prefix}file_2", @temp_object)
        @data_store.store(@temp_object)
      end
    
      it "should use a different filename taking into account the name and ext" do
        @temp_object.name = 'hello.png'
        touch_file("#{@file_pattern_prefix}hello.png")
        @data_store.should_receive(:disambiguate).with('hello.png').and_return('blah.png')
        @data_store.store(@temp_object)
      end

      it "should keep trying until it finds a free filename" do
        touch_file("#{@file_pattern_prefix}file")
        touch_file("#{@file_pattern_prefix}file_2")
        @data_store.should_receive(:disambiguate).with('file').and_return('file_2')
        @data_store.should_receive(:disambiguate).with('file_2').and_return('file_3')
        it_should_write_to_file("#{@file_pattern_prefix}file_3", @temp_object)
        @data_store.store(@temp_object)
      end

    end

    describe "return value" do

      it "should return the filepath without the root of the stored file when a file name is not provided" do
        @data_store.store(@temp_object).should == "#{@file_pattern_prefix_without_root}file"
      end
    
      it "should return the filepath without the root of the stored file when a file name is provided" do
        @temp_object.name = 'hello.you.png'
        @data_store.store(@temp_object).should == "#{@file_pattern_prefix_without_root}hello.you.png"
      end
    
    end

  end
  
  describe "disambiguate" do
    it "should add a suffix" do
      @data_store.disambiguate('file').should =~ /^file_\w+$/
    end
    it "should add a suffix to the basename" do
      @data_store.disambiguate('file.png').should =~ /^file_\w+\.png$/
    end
    it "should be random(-ish)" do
      @data_store.disambiguate('file').should_not == @data_store.disambiguate('file')
    end
  end
  
  describe "errors" do

    it "should raise an error if it can't create a directory" do
      FileUtils.should_receive(:mkdir_p).and_raise(Errno::EACCES)
      lambda{ @data_store.store(@temp_object) }.should raise_error(Dragonfly::DataStorage::UnableToStore)
    end
  
    it "should raise an error if it can't create a file" do
      @temp_object.should_receive(:to_file).and_raise(Errno::EACCES)
      lambda{ @data_store.store(@temp_object) }.should raise_error(Dragonfly::DataStorage::UnableToStore)
    end

  end
  
  describe "retrieve" do
    it "should be able to retrieve any file, stored or not (and without extra data)" do
      FileUtils.mkdir_p("#{@data_store.root_path}/jelly_beans/are")
      File.open("#{@data_store.root_path}/jelly_beans/are/good", 'w'){|f| f.write('hey dog') }
      file, meta = @data_store.retrieve("jelly_beans/are/good")
      file.read.should == 'hey dog'
      meta.should == {}
    end
  end
  
  describe "destroying" do

    it "should raise an error if the data doesn't exist" do
      lambda{
        @data_store.destroy('gooble/gubbub')
      }.should raise_error(Dragonfly::DataStorage::DataNotFound)
    end

    it "should prune empty directories when destroying" do
      uid = @data_store.store(@temp_object)
      @data_store.destroy(uid)
{'fraction_non_alphanumeric': 0.11698257706299062, 'fraction_numerical': 0.0047865211564235115, 'mean_word_length': 3.5759964958388086, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3865792', 'n_tokens_mistral': 3474, 'n_tokens_neox': 3388, 'n_words': 735}