<commit_before>module Attache
  class << self
    attr_accessor :localdir,
                  :remotedir,
                  :storage,
                  :cache,
                  :logger,
                  :bucket,
                  :geometry_alias,
                  :secret_key
  end
end

Attache.logger     = Logger.new(STDOUT)
Attache.localdir   = File.expand_path(ENV.fetch('LOCAL_DIR') { Dir.mktmpdir })
Attache.remotedir  = ENV['REMOTE_DIR'] # nil means no fixed top level remote directory, and that's fine.
Attache.secret_key = ENV['SECRET_KEY'] # nil means no auth check; anyone can upload a file

Attache.geometry_alias = JSON.parse(ENV.fetch('GEOMETRY_ALIAS') { '{}' })
# e.g. GEOMETRY_ALIAS='{ "small": "64x64#", "large": "128x128x#" }'

if ENV['FOG_CONFIG'] && (config = JSON.parse(ENV['FOG_CONFIG']))
  Attache.storage = Fog::Storage.new(config.symbolize_keys)
  Attache.bucket  = config['s3_bucket']
end

if Attache.storage
  # lru eviction only when there is Attache.storage
  Attache.cache = DiskStore.new(Attache.localdir, {
    cache_size: ENV.fetch('CACHE_SIZE_BYTES') {
      stat = Sys::Filesystem.stat("/")
      available = stat.block_size * stat.blocks_available
      (available * 0.8).floor # use 80% free disk by default
    },
    reaper_interval:   ENV.fetch('CACHE_EVICTION_INTERVAL_SECONDS') { 60 },
    eviction_strategy: :LRU,
  })
else
  Attache.cache = DiskStore.new(Attache.localdir)
end
<commit_msg>Add bootup config info in logs<commit_after>module Attache
  class << self
    attr_accessor :localdir,
                  :remotedir,
                  :storage,
                  :cache,
                  :logger,
                  :bucket,
                  :geometry_alias,
                  :secret_key
  end
end

Attache.logger     = Logger.new(STDOUT)
Attache.localdir   = File.expand_path(ENV.fetch('LOCAL_DIR') { Dir.mktmpdir })
Attache.remotedir  = ENV['REMOTE_DIR'] # nil means no fixed top level remote directory, and that's fine.
Attache.secret_key = ENV['SECRET_KEY'] # nil means no auth check; anyone can upload a file

Attache.geometry_alias = JSON.parse(ENV.fetch('GEOMETRY_ALIAS') { '{}' })
# e.g. GEOMETRY_ALIAS='{ "small": "64x64#", "large": "128x128x#" }'

if ENV['FOG_CONFIG'] && (config = JSON.parse(ENV['FOG_CONFIG']))
  Attache.storage = Fog::Storage.new(config.symbolize_keys)
  Attache.bucket  = config['s3_bucket']
end

if Attache.storage
  # lru eviction only when there is Attache.storage
  Attache.cache = DiskStore.new(Attache.localdir, {
    cache_size: ENV.fetch('CACHE_SIZE_BYTES') {
      stat = Sys::Filesystem.stat("/")
      available = stat.block_size * stat.blocks_available
      (available * 0.8).floor # use 80% free disk by default
    },
    reaper_interval:   ENV.fetch('CACHE_EVICTION_INTERVAL_SECONDS') { 60 },
    eviction_strategy: :LRU,
  })
else
  Attache.cache = DiskStore.new(Attache.localdir)
end

# summary
Attache.logger.info({
  storage: Attache.storage.class.name,
  secret_key: !!Attache.secret_key,
  cache: Attache.cache,
})

{'fraction_non_alphanumeric': 0.1270681667769689, 'fraction_numerical': 0.011250827266710787, 'mean_word_length': 3.0414438502673797, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5781259', 'n_tokens_mistral': 1085, 'n_tokens_neox': 1022, 'n_words': 207}