<commit_before># Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
#
# This program is licensed to you under the Apache License Version 2.0,
# and you may not use this file except in compliance with the Apache License Version 2.0.
# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the Apache License Version 2.0 is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.

# Author::    <NAME> (mailto:<EMAIL>)
# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
# License::   Apache License Version 2.0

require 'optparse'
require 'date'
require 'yaml'

# Config module to hold functions related to CLI argument parsing
# and config file reading to support the daily ETL job.
module InfobrightLoader
  module Cli
    module Config

      # What are we called?
      SCRIPT_NAME = InfobrightLoader::NAME

      # For errors
      class ConfigError < ArgumentError; end

      # Configuration for loading all the files from a specific directory into
      # a specific table
      LoadHashConfig = Struct.new(:load_hash, :db, :processes, :separator, :encloser)
      
      # Configuration for loading a set of tables from a set of files (where
      # each table can have multiple files loaded into it)
      LoadFolderConfig = Struct.new(:folder, :table, :db, :processes, :separator, :encloser)

      # Validates and returns the configuration.
      #
      # The configuration returned will either be
      # a LoadMapConfig or a LoadFolderConfig.
      def get_config()

        options = Config.parse_args()

        if options[:control].nil?

          config = LoadFolderConfig.new
          config.processes = options[:processes]
          config.db = options[:db] 
          config.separator = options[:separator]
          config.encloser = options[:encloser] 
          config.folder = options[:folder] 
          config.table = options[:table] 

        else

          yaml = YAML.load_file(options[:control])

          # Set the overridable fields if they haven't been overridden at the command-line
          config = LoadMapConfig.new
          get_or_else = lambda {|x, y| x.nil? ? y : x }
          config.processes = get_or_else.call(options[:processes], yaml[:load][:processes])
          config.db = get_or_else.call(options[:db], yaml[:database][:name])
          config.separator = get_or_else.call(options[:separator], yaml[:data_format][:separator])
          config.encloser = get_or_else.call(options[:encloser], yaml[:data_format][:encloser])

          # Finally grab the load map
          config.load_hash = yaml[:data_loads]
        end

        # Finally we can check that number of processes is a positive integer
        unless config.processes.to_i > 0
          raise ConfigError, "Parallel load processes '#{config.processes}' is not a positive integer"
        end
        config.processes = config.processes.to_i # A kitten dies, mutably.

        config # Return either our LoadFolderConfig or our LoadMapConfig
      end
      module_function :get_config<commit_msg>Fixed typo<commit_after># Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
#
# This program is licensed to you under the Apache License Version 2.0,
# and you may not use this file except in compliance with the Apache License Version 2.0.
# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the Apache License Version 2.0 is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.

# Author::    <NAME> (mailto:<EMAIL>)
# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
# License::   Apache License Version 2.0

require 'optparse'
require 'date'
require 'yaml'

# Config module to hold functions related to CLI argument parsing
# and config file reading to support the daily ETL job.
module InfobrightLoader
  module Cli
    module Config

      # What are we called?
      SCRIPT_NAME = InfobrightLoader::NAME

      # For errors
      class ConfigError < ArgumentError; end

      # Configuration for loading all the files from a specific directory into
      # a specific table
      LoadHashConfig = Struct.new(:load_hash, :db, :processes, :separator, :encloser)
      
      # Configuration for loading a set of tables from a set of files (where
      # each table can have multiple files loaded into it)
      LoadFolderConfig = Struct.new(:folder, :table, :db, :processes, :separator, :encloser)

      # Validates and returns the configuration.
      #
      # The configuration returned will either be
      # a LoadMapConfig or a LoadFolderConfig.
      def get_config()

        options = Config.parse_args()

        if options[:control].nil?

          config = LoadFolderConfig.new
          config.processes = options[:processes]
          config.db = options[:db] 
          config.separator = options[:separator]
          config.encloser = options[:encloser] 
          config.folder = options[:folder] 
          config.table = options[:table] 

        else

          yaml = YAML.load_file(options[:control])

          # Set the overridable fields if they haven't been overridden at the command-line
          config = LoadHashConfig.new
          get_or_else = lambda {|x, y| x.nil? ? y : x }
          config.processes = get_or_else.call(options[:processes], yaml[:load][:processes])
          config.db = get_or_else.call(options[:db], yaml[:database][:name])
          config.separator = get_or_else.call(options[:separator], yaml[:data_format][:separator])
          config.encloser = get_or_else.call(options[:encloser], yaml[:data_format][:encloser])

          # Finally grab the load map
          config.load_hash = yaml[:data_loads]
        end

        # Finally we can check that number of processes is a positive integer
        unless config.processes.to_i > 0
          raise ConfigError, "Parallel load processes '#{config.processes}' is not a positive integer"
        end
        config.processes = config.processes.to_i # A kitten dies, mutably.

        config # Return either our LoadFolderConfig or our LoadHashConfig
      end
      module_function :get_config
{'fraction_non_alphanumeric': 0.08087464430133294, 'fraction_numerical': 0.006889321551595028, 'mean_word_length': 2.998802395209581, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5507440', 'n_tokens_mistral': 1847, 'n_tokens_neox': 1742, 'n_words': 670}