<commit_before>require 'singleton'
require 'gidget/page'
require 'gidget/post'


module Gidget
  class FileMapper
    include Singleton
  
  
    def initialize
      @pages = Hash.new
      @posts = Array.new
    end
    
    
    def pages
      @pages
    end


    def posts
      if (date_last_loaded.day != DateTime.now.day) load_posts
      @posts
    end

    
    def load
      load_pages
      load_posts
    end


    private
    def load_pages
      page_paths = Dir.glob("pages/**/*.txt")

      # load the page index
      page_paths.each { |file_path|
        page = Page.new(file_path)
        @pages[page.request_path] = page
      }
      
      puts "Page Index created, size = " + @pages.keys.size.to_s
    end


   def load_posts
      posts.clear
      date_last_loaded = DateTime.now

      post_paths = Dir.glob("posts/**/*.txt")

      # load the post index
      post_paths.each { |file_path|
        post = Post.new(file_path)
        @posts << post
      }
      
      # sort the post index by date descending (newest will be first)
      @posts.replace @posts.sort_by { |p| Time.parse(p.date.to_s).to_i }.reverse!
      
      puts "Post Index created, size = " + @posts.size.to_s
    end
  end
end<commit_msg>added expiration of the posts array within the file_mapper to allow scheduled posting<commit_after>require 'singleton'
require 'gidget/page'
require 'gidget/post'


module Gidget
  class FileMapper
    include Singleton
  
  
    def initialize
      @pages = Hash.new
      @posts = Array.new
    end
    
    
    def pages
      @pages
    end


    def posts
      if (@date_last_loaded.day != DateTime.now.day)
        load_posts
      end

      @posts
    end

    
    def load
      load_pages
      load_posts
    end


    private
    def load_pages
      page_paths = Dir.glob("pages/**/*.txt")

      # load the page index
      page_paths.each { |file_path|
        page = Page.new(file_path)
        @pages[page.request_path] = page
      }
      
      puts "Page Index created, size = " + @pages.keys.size.to_s
    end


    def load_posts
      @posts.clear
      @date_last_loaded = DateTime.now

      post_paths = Dir.glob("posts/**/*.txt")

      # load the post index
      post_paths.each { |file_path|
        post = Post.new(file_path)
	
        if (post.date < @date_last_loaded)
          @posts << post
        end
      }
      
      # sort the post index by date descending (newest will be first)
      @posts.replace @posts.sort_by { |p| Time.parse(p.date.to_s).to_i }.reverse!
      
      puts "Post Index created, size = " + @posts.size.to_s
    end
  end
end
{'fraction_non_alphanumeric': 0.09180576631259484, 'fraction_numerical': 0.0, 'mean_word_length': 1.8914473684210527, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5492515', 'n_tokens_mistral': 869, 'n_tokens_neox': 844, 'n_words': 216}