<commit_before># -*- coding: utf-8 -*-

require 'etc'
require 'json'
require 'logger'
require 'logger/joint'
require 'pathname'
require 'riser'
require 'socket'
require 'yaml'

module RIMS
  class Service
    class Configuration
      class << self
        def stringify_symbol(collection)
          case (collection)
          when Hash
            Hash[collection.map{|key, value| [ stringify_symbol(key), stringify_symbol(value) ] }]
          when Array
            collection.map{|i| stringify_symbol(i) }
          else
            case (value = collection)
            when Symbol
              value.to_s
            else
              value
            end
          end
        end

        def update(dest, other)
          case (dest)
          when Hash
            unless (other.is_a? Hash) then
              raise ArgumentError, 'hash can only be updated with hash.'
            end
            for key, value in other
              dest[key] = update(dest[key], value)
            end
            dest
          when Array
            if (other.is_a? Array) then
              dest.concat(other)
            else
              other
            end
          else
            other
          end
        end

        def get_configuration(collection, base_dir)
          if ((collection.key? 'configuration') && (collection.key? 'configuration_file')) then
            raise KeyError, 'configuration conflict: configuration, configuraion_file'
          end

          if (collection.key? 'configuration') then
            collection['configuration']
          elsif (collection.key? 'configuration_file') then
            configuration_file_path = Pathname(collection['configuration_file'])
            if (configuration_file_path.relative?) then
              configuration_file_path = base_dir + configuration_file_path # expect base_dir to be Pathname
            end
            YAML.load_file(configuration_file_path.to_s)
          else
            {}
          end
        end
      end

      def initialize
        @config = {}
      end

      def load(config, load_path=nil)
        stringified_config = self.class.stringify_symbol(config)
        if (stringified_config.key? 'base_dir') then
          base_dir = Pathname(stringified_config['base_dir'])
          if (load_path && base_dir.relative?) then
            stringified_config['base_dir'] = Pathname(load_path) + base_dir
          else
            stringified_config['base_dir'] = base_dir
          end
        elsif (load_path) then
          stringified_config['base_dir'] = Pathname(load_path)
        end
        self.class.update(@config, stringified_config)

        self
      end

      # configuration example.
      #   required_features:
      #     - rims/qdbm
      #     - rims/passwd/ldap
      #   logging:
      #     file:
      #       path: rims.log
      #       shift_age: 10
      #       shift_size: 1048576
      #       level: debug
      #       datetime_format: %Y-%m-%d %H:%M:%S
      #       shift_period_suffix: %Y%m%d
      #     stdout:
      #       level: info
      #       datetime_format: %Y-%m-%d %H:%M:%S
      #   server:
      #     listen_address:
      #       # see `Riser::SocketAddress.parse' for address format
      #       type: tcp
      #       host: 0.0.0.0
      #       port: 143
      #       backlog: 64
      #     accept_polling_timeout_seconds: 0.1
      #     thread_num: 20
      #     thread_queue_size: 20
      #     thread_queue_polling_timeout_seconds: 0.1
      #     send_buffer_limit_size: 16384
      #   lock:
      #     read_lock_timeout_seconds: 30
      #     write_lock_timeout_seconds: 30
      #     cleanup_write_lock_timeout_seconds: 1
      #   storage:
      #     meta_key_value_store:
      #       type: qdbm_depot
      #       configuration:
      #         bnum 1200000
      #       use_checksum: true
      #     text_key_value_store:
      #       type: qdbm_curia
      #       configuration_file: text_kvs_config.yml
      #       use_checksum: true
      #   authentication:
      #     hostname: imap.example.com
      #     password_sources:
      #       - type: plain
      #         configuration:
      #           - user: alice
      #             pass: <PASSWORD>
      #           - user: bob
      #             pass: <PASSWORD>
      #       - type: hash
      #         configuration_file: passwd_hash.yml
      #       - type: ldap
      #         configuration:
      #           ldap_uri: ldap://ldap.example.com/ou=user,o=example,dc=nodomain?uid?one?(memberOf=cn=imap,ou=group,o=example,dc=nodomain)
      #   authorization:
      #     mail_delivery_user: "#postman"
      def load_yaml(path)
        load(YAML.load_file(path), File.dirname(path))
        self
      end

      def require_features
        if (feature_list = @config.dig('required_features')) then
          for feature in feature_list
            require(feature)
          end
        end

        nil
      end
<commit_msg>service configuration: define built-in plug-in at require_features method<commit_after># -*- coding: utf-8 -*-

require 'etc'
require 'json'
require 'logger'
require 'logger/joint'
require 'pathname'
require 'riser'
require 'socket'
require 'yaml'

module RIMS
  class Service
    class Configuration
      class << self
        def stringify_symbol(collection)
          case (collection)
          when Hash
            Hash[collection.map{|key, value| [ stringify_symbol(key), stringify_symbol(value) ] }]
          when Array
            collection.map{|i| stringify_symbol(i) }
          else
            case (value = collection)
            when Symbol
              value.to_s
            else
              value
            end
          end
        end

        def update(dest, other)
          case (dest)
          when Hash
            unless (other.is_a? Hash) then
              raise ArgumentError, 'hash can only be updated with hash.'
            end
            for key, value in other
              dest[key] = update(dest[key], value)
            end
            dest
          when Array
            if (other.is_a? Array) then
              dest.concat(other)
            else
              other
            end
          else
            other
          end
        end

        def get_configuration(collection, base_dir)
          if ((collection.key? 'configuration') && (collection.key? 'configuration_file')) then
            raise KeyError, 'configuration conflict: configuration, configuraion_file'
          end

          if (collection.key? 'configuration') then
            collection['configuration']
          elsif (collection.key? 'configuration_file') then
            configuration_file_path = Pathname(collection['configuration_file'])
            if (configuration_file_path.relative?) then
              configuration_file_path = base_dir + configuration_file_path # expect base_dir to be Pathname
            end
            YAML.load_file(configuration_file_path.to_s)
          else
            {}
          end
        end
      end

      def initialize
        @config = {}
      end

      def load(config, load_path=nil)
        stringified_config = self.class.stringify_symbol(config)
        if (stringified_config.key? 'base_dir') then
          base_dir = Pathname(stringified_config['base_dir'])
          if (load_path && base_dir.relative?) then
            stringified_config['base_dir'] = Pathname(load_path) + base_dir
          else
            stringified_config['base_dir'] = base_dir
          end
        elsif (load_path) then
          stringified_config['base_dir'] = Pathname(load_path)
        end
        self.class.update(@config, stringified_config)

        self
      end

      # configuration example.
      #   required_features:
      #     - rims/qdbm
      #     - rims/passwd/ldap
      #   logging:
      #     file:
      #       path: rims.log
      #       shift_age: 10
      #       shift_size: 1048576
      #       level: debug
      #       datetime_format: %Y-%m-%d %H:%M:%S
      #       shift_period_suffix: %Y%m%d
      #     stdout:
      #       level: info
      #       datetime_format: %Y-%m-%d %H:%M:%S
      #   server:
      #     listen_address:
      #       # see `Riser::SocketAddress.parse' for address format
      #       type: tcp
      #       host: 0.0.0.0
      #       port: 143
      #       backlog: 64
      #     accept_polling_timeout_seconds: 0.1
      #     thread_num: 20
      #     thread_queue_size: 20
      #     thread_queue_polling_timeout_seconds: 0.1
      #     send_buffer_limit_size: 16384
      #   lock:
      #     read_lock_timeout_seconds: 30
      #     write_lock_timeout_seconds: 30
      #     cleanup_write_lock_timeout_seconds: 1
      #   storage:
      #     meta_key_value_store:
      #       type: qdbm_depot
      #       configuration:
      #         bnum 1200000
      #       use_checksum: true
      #     text_key_value_store:
      #       type: qdbm_curia
      #       configuration_file: text_kvs_config.yml
      #       use_checksum: true
      #   authentication:
      #     hostname: imap.example.com
      #     password_sources:
      #       - type: plain
      #         configuration:
      #           - user: alice
      #             pass: <PASSWORD>
      #           - user: bob
      #             pass: <PASSWORD>
      #       - type: hash
      #         configuration_file: passwd_hash.yml
      #       - type: ldap
      #         configuration:
      #           ldap_uri: ldap://ldap.example.com/ou=user,o=example,dc=nodomain?uid?one?(memberOf=cn=imap,ou=group,o=example,dc=nodomain)
      #   authorization:
      #     mail_delivery_user: "#postman"
      def load_yaml(path)
        load(YAML.load_file(path), File.dirname(path))
        self
      end

      def require_features
        # built-in plug-in
        require 'rims/gdbm_kvs'
        require 'rims/passwd'

        if (feature_list = @config.dig('required_features')) then
          for feature in feature_list
            require(feature)
          end
        end

        nil
      end

{'fraction_non_alphanumeric': 0.08438485804416404, 'fraction_numerical': 0.008675078864353312, 'mean_word_length': 1.577489837398374, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1027013', 'n_tokens_mistral': 2967, 'n_tokens_neox': 2867, 'n_words': 613}