<commit_before>module Spider; module Model; module Storage
    
    class BaseStorage
        include Spider::Logger
        attr_reader :url
        
        def self.sequence_sync
            @sequence_sync ||= Sync.new
        end
        
        def self.base_types
            Model.base_types
        end
        
        def initialize(url)
            @url = url
            @configuration = {}
            parse_url(url)
        end
        
        def configure(conf)
            @configuration.merge!(conf)
        end
        
        def parse_url(url)
            raise StorageException, "Unimplemented"
        end
        
        def get_mapper(model)
            raise StorageException, "Unimplemented"
        end
        
        def prepare_value(type, value)
            return value
        end
        
        def ==(storage)
            return false unless self.class == storage.class
            return false unless self.url == storage.url
            return true
        end
        
        # Utility methods
        
        def sequence_file_path(name)
            path = 'var/sequences/'+name
            return path
        end
        
        def sequence_exists?(name)
            File.exist?(sequence_file_path(name))
        end
        
        def create_sequence(name, start=1, increment=1)
            sequence_next(name, start-1, increment)
        end
            
        
        def update_sequence(name, val)
            # not an alias because the set value behaviour of next_sequence isn't expected in subclasses
            sequence_next(name, val)
        end
        
        # Increments a named sequence and returns the new value
        def sequence_next(name, newval=nil, increment=1)
            path = sequence_file_path(name)
            FileUtils.mkpath(File.dirname(path))
            self.class.sequence_sync.lock(Sync::EX)
            if newval
                seq = newval
            else
                seq = 0
                File.open(path, 'a+') do |f|
                    f.rewind
                    f.flock File::LOCK_EX
                    cur = f.gets
                    if (cur)
                        seq, increment_str = cur.split('|')
                    else
                        seq, increment_str = 0, 1
                    end
                    seq = seq.to_i
                    increment = increment_str.to_i if increment_str
                    f.close
                end
                seq += increment
            end
            File.open(path, 'w+') do |f|
                f.print(seq)
                f.print("|#{increment}") if (increment != 1)
                f.flock File::LOCK_UN
                f.close
            end
            self.class.sequence_sync.lock(Sync::UN)
            return seq
        end
            
        
    end
    
    ###############################
    #   Exceptions                #
    ###############################
    
    class StorageException < RuntimeError
    end
    
    
    
end; end; end<commit_msg>Fix for new storage configuration format<commit_after>module Spider; module Model; module Storage
    
    class BaseStorage
        include Spider::Logger
        attr_reader :url
        
        def self.sequence_sync
            @sequence_sync ||= Sync.new
        end
        
        def self.base_types
            Model.base_types
        end
        
        def initialize(url)
            @url = url
            @configuration = {}
            parse_url(url)
        end
        
        def configure(conf)
            @configuration.merge!(conf.to_hash)
        end
        
        def parse_url(url)
            raise StorageException, "Unimplemented"
        end
        
        def get_mapper(model)
            raise StorageException, "Unimplemented"
        end
        
        def prepare_value(type, value)
            return value
        end
        
        def ==(storage)
            return false unless self.class == storage.class
            return false unless self.url == storage.url
            return true
        end
        
        # Utility methods
        
        def sequence_file_path(name)
            path = 'var/sequences/'+name
            return path
        end
        
        def sequence_exists?(name)
            File.exist?(sequence_file_path(name))
        end
        
        def create_sequence(name, start=1, increment=1)
            sequence_next(name, start-1, increment)
        end
            
        
        def update_sequence(name, val)
            # not an alias because the set value behaviour of next_sequence isn't expected in subclasses
            sequence_next(name, val)
        end
        
        # Increments a named sequence and returns the new value
        def sequence_next(name, newval=nil, increment=1)
            path = sequence_file_path(name)
            FileUtils.mkpath(File.dirname(path))
            self.class.sequence_sync.lock(Sync::EX)
            if newval
                seq = newval
            else
                seq = 0
                File.open(path, 'a+') do |f|
                    f.rewind
                    f.flock File::LOCK_EX
                    cur = f.gets
                    if (cur)
                        seq, increment_str = cur.split('|')
                    else
                        seq, increment_str = 0, 1
                    end
                    seq = seq.to_i
                    increment = increment_str.to_i if increment_str
                    f.close
                end
                seq += increment
            end
            File.open(path, 'w+') do |f|
                f.print(seq)
                f.print("|#{increment}") if (increment != 1)
                f.flock File::LOCK_UN
                f.close
            end
            self.class.sequence_sync.lock(Sync::UN)
            return seq
        end
            
        
    end
    
    ###############################
    #   Exceptions                #
    ###############################
    
    class StorageException < RuntimeError
    end
    
    
    
end; end; end
{'fraction_non_alphanumeric': 0.08424311552875997, 'fraction_numerical': 0.002607137037640541, 'mean_word_length': 1.1424083769633508, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1777164', 'n_tokens_mistral': 1560, 'n_tokens_neox': 1509, 'n_words': 378}