<commit_before># rSmart client library and command-line tool to help interact with rSmart's cloud APIs.
# Copyright (C) 2014 The rSmart Group, Inc.

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

require "rsmart_toolbox"

module Rsmart::ETL

  class TextParseError < StandardError
  end

  # @return [Exception] an Exception with a message formatted with $INPUT_LINE_NUMBER.
  # @param [String, Exception] e the error to handle
  def self.error(e)
    if e.kind_of? String
      # default to TextParseError
      return TextParseError.new "ERROR: Line #{$INPUT_LINE_NUMBER}: #{e}"
    end
    if e.kind_of? Exception
      return e.exception "ERROR: Line #{$INPUT_LINE_NUMBER}: #{e}"
    end
    raise ArgumentError, "Unsupported error type: #{e.class}"
  end

  # @return [Exception] an Exception with a message formatted with $INPUT_LINE_NUMBER.
  # @param [String, Exception] e the warning to handle
  def self.warning(e)
    if e.kind_of? String
      # default to TextParseError
      return TextParseError.new "WARN:  Line #{$INPUT_LINE_NUMBER}: #{e}"
    end
    if e.kind_of? Exception
      return e.exception "WARN:  Line #{$INPUT_LINE_NUMBER}: #{e}"
    end
    raise ArgumentError, "Unsupported error type: #{e.class}"
  end

  # @param [String, #match] subject used for validity checking.
  # @param [Array<Object>, Regexp] valid_values all of the possible valid values.
  # @option opt [Boolean] :case_sensitive performs case sensitive matching
  # @return [Boolean] true if the subject matches valid_values.
  #   FYI valid_values must respond to #casecmp.
  # @raise [ArgumentError] if valid_values is nil or empty.
  # @raise [ArgumentError] case sensitive matching only works for objects
  #   that respond to #casecmp; primarily String objects.
  def self.valid_value(subject, valid_values, opt={})
    raise ArgumentError, "valid_values must not be nil!" if valid_values.nil?
    if valid_values.kind_of? Regexp
      return true if subject =~ valid_values
    end
    if valid_values.kind_of? Array
      raise ArgumentError, "valid_values must have at least one element!" unless valid_values.length > 0
      if opt[:case_sensitive] == false # case insensitive comparison requested
        raise ArgumentError, "Object must respond to #casecmp" unless subject.respond_to? 'casecmp'
        valid_values.each do |valid_value|
          return true if valid_value.casecmp(subject) == 0
        end
      end
      return true if valid_values.include? subject # default to == equality
    end
    return false
  end

  # @param [String] str String to be matched against well known boolean patterns.
  # @option opt [Boolean] :default the default return value if str is empty.
  # @return [Boolean] the result of matching the str input against well known boolean patterns.
  # @raise [TextParseError] if none of the known boolean patterns could be matched.
  def self.parse_boolean(str, opt={})
    return true  if str == true
    return false if str == false
    b = parse_string str, opt
    return true  if b =~ /^(active|a|true|t|yes|y|1)$/i
    return false if b =~ /^(inactive|i|false|f|no|n|0)$/i
    if b.empty? && !opt[:default].nil?
      return opt[:default]
    end
    if b.empty?
      return nil
    end
    raise Rsmart::ETL::error TextParseError.new "invalid value for Boolean: '#{str}'"
  end

  # @param [String] str the String to be encoded and invalid characters replaced with valid characters.
  # @option opt [String] :encoding the character encoding to use.
  # @return [String] the result of encoding the String and replacing invalid characters with valid characters.
  def self.encode(str, opt={ encoding: "UTF-8" } )
    opt[:encoding] = "UTF-8" if opt[:encoding].nil?
    str.encode( opt[:encoding], :invalid => :replace,
                :undef => :replace, :replace => "" )
  end

  # Matches MRI CSV specification:
  # The header String is downcased, spaces are replaced with underscores,
  # non-word characters are dropped, and finally to_sym() is called.
  def self.to_symbol(str)
    raise ArgumentError, "Illegal symbol name: '#{str}'" if str.nil? || str.empty?
    encode( str.downcase.gsub(/\s+/, "_").gsub(/\W+/, "") ).to_sym
  end

  # DRY up some common string manipulation
  def self.mutate_sql_stmt!(insert_str, column_name, values_str, value)
    insert_str.concat "#{column_name.upcase},"
    # TODO what are all of the valid types that should not be quoted?
    if value.kind_of? Integer
      values_str.concat "#{value},"
    else
      values_str.concat "'#{value}',"
    end
  end

  def self.escape_single_quotes(str)
    if str.nil?
      return nil
    end
    return str.to_s.gsub("'", "\\\\'")
  end

  def self.parse_string(str, opt={})
    opt[:strict]   = true if opt[:strict].nil?
    retval = encode str.to_s.strip
    if opt[:required] && retval.empty?
      raise Rsmart::ETL::error TextParseError.new "Required data element '#{opt[:name]}' not found: '#{str}'"
    end<commit_msg>document the to_symbol method<commit_after># rSmart client library and command-line tool to help interact with rSmart's cloud APIs.
# Copyright (C) 2014 The rSmart Group, Inc.

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

require "rsmart_toolbox"

module Rsmart::ETL

  class TextParseError < StandardError
  end

  # @return [Exception] an Exception with a message formatted with $INPUT_LINE_NUMBER.
  # @param [String, Exception] e the error to handle
  def self.error(e)
    if e.kind_of? String
      # default to TextParseError
      return TextParseError.new "ERROR: Line #{$INPUT_LINE_NUMBER}: #{e}"
    end
    if e.kind_of? Exception
      return e.exception "ERROR: Line #{$INPUT_LINE_NUMBER}: #{e}"
    end
    raise ArgumentError, "Unsupported error type: #{e.class}"
  end

  # @return [Exception] an Exception with a message formatted with $INPUT_LINE_NUMBER.
  # @param [String, Exception] e the warning to handle
  def self.warning(e)
    if e.kind_of? String
      # default to TextParseError
      return TextParseError.new "WARN:  Line #{$INPUT_LINE_NUMBER}: #{e}"
    end
    if e.kind_of? Exception
      return e.exception "WARN:  Line #{$INPUT_LINE_NUMBER}: #{e}"
    end
    raise ArgumentError, "Unsupported error type: #{e.class}"
  end

  # @param [String, #match] subject used for validity checking.
  # @param [Array<Object>, Regexp] valid_values all of the possible valid values.
  # @option opt [Boolean] :case_sensitive performs case sensitive matching
  # @return [Boolean] true if the subject matches valid_values.
  #   FYI valid_values must respond to #casecmp.
  # @raise [ArgumentError] if valid_values is nil or empty.
  # @raise [ArgumentError] case sensitive matching only works for objects
  #   that respond to #casecmp; primarily String objects.
  def self.valid_value(subject, valid_values, opt={})
    raise ArgumentError, "valid_values must not be nil!" if valid_values.nil?
    if valid_values.kind_of? Regexp
      return true if subject =~ valid_values
    end
    if valid_values.kind_of? Array
      raise ArgumentError, "valid_values must have at least one element!" unless valid_values.length > 0
      if opt[:case_sensitive] == false # case insensitive comparison requested
        raise ArgumentError, "Object must respond to #casecmp" unless subject.respond_to? 'casecmp'
        valid_values.each do |valid_value|
          return true if valid_value.casecmp(subject) == 0
        end
      end
      return true if valid_values.include? subject # default to == equality
    end
    return false
  end

  # @param [String] str String to be matched against well known boolean patterns.
  # @option opt [Boolean] :default the default return value if str is empty.
  # @return [Boolean] the result of matching the str input against well known boolean patterns.
  # @raise [TextParseError] if none of the known boolean patterns could be matched.
  def self.parse_boolean(str, opt={})
    return true  if str == true
    return false if str == false
    b = parse_string str, opt
    return true  if b =~ /^(active|a|true|t|yes|y|1)$/i
    return false if b =~ /^(inactive|i|false|f|no|n|0)$/i
    if b.empty? && !opt[:default].nil?
      return opt[:default]
    end
    if b.empty?
      return nil
    end
    raise Rsmart::ETL::error TextParseError.new "invalid value for Boolean: '#{str}'"
  end

  # @param [String] str the String to be encoded and invalid characters replaced with valid characters.
  # @option opt [String] :encoding the character encoding to use.
  # @return [String] the result of encoding the String and replacing invalid characters with valid characters.
  def self.encode(str, opt={ encoding: "UTF-8" } )
    opt[:encoding] = "UTF-8" if opt[:encoding].nil?
    str.encode( opt[:encoding], :invalid => :replace,
                :undef => :replace, :replace => "" )
  end

  # Matches the MRI CSV specification:
  # The header String is downcased, spaces are replaced with underscores,
  # non-word characters are dropped, and finally to_sym() is called.
  # @param [String] str the String to be symbolized.
  # @return [Symbol] String is downcased, spaces are replaced with underscores,
  #   non-word characters are dropped
  # @raise [ArgumentError] if str is nil or empty.
  def self.to_symbol(str)
    raise ArgumentError, "Illegal symbol name: '#{str}'" if str.nil? || str.empty?
    encode( str.downcase.gsub(/\s+/, "_").gsub(/\W+/, "") ).to_sym
  end

  # DRY up some common string manipulation
  def self.mutate_sql_stmt!(insert_str, column_name, values_str, value)
    insert_str.concat "#{column_name.upcase},"
    # TODO what are all of the valid types that should not be quoted?
    if value.kind_of? Integer
      values_str.concat "#{value},"
    else
      values_str.concat "'#{value}',"
    end
  end

  def self.escape_single_quotes(str)
    if str.nil?
      return nil
    end
    return str.to_s.gsub("'", "\\\\'")
  end

  def self.parse_string(str, opt={})
    opt[:strict]   = true if opt[:strict].nil?
    retval = encode str.to_s.strip
    if opt[:required] && retval.empty?
      raise Rsmart::ETL::error TextParseError.new "Required data element '#{opt[:name]}' not found: '#{str}'"
    end
{'fraction_non_alphanumeric': 0.09880450070323488, 'fraction_numerical': 0.0019338959212376935, 'mean_word_length': 3.5362838915470496, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7454768', 'n_tokens_mistral': 3369, 'n_tokens_neox': 3299, 'n_words': 1299}