<commit_before>
# frozen_string_literal: true
require_relative 'matching'
require_relative 'roman'
require_relative 'fuzzy_matcher'
require_relative 'cases'
require_relative 'regexp'

module BBLib
  # Quickly remove any symbols from a string leaving only alpha-numeric characters and white space.
  def self.drop_symbols(str)
    str.gsub(/[^\w\s\d]|_/, '')
  end

  # Extract all integers from a string. Use extract_floats if numbers may contain decimal places.
  def self.extract_integers(str, convert: true)
    BBLib.extract_numbers(str, convert: false).reject { |r| r.include?('.') }
         .map { |m| convert ? m.to_i : m }
  end

  # Extracts all integers or decimals from a string into an array.
  def self.extract_floats(str, convert: true)
    BBLib.extract_numbers(str, convert: false).reject { |r| !r.include?('.') }
         .map { |m| convert ? m.to_f : m }
  end

  # Extracts any correctly formed integers or floats from a string
  def self.extract_numbers(str, convert: true)
    str.scan(/\d+\.\d+(?<=[^\.])|\d+(?<=[^\.])|\d+\.\d+$|\d+$/)
       .map { |f| convert ? (f.include?('.') ? f.to_f : f.to_i) : f }
  end

  # Used to move the position of the articles 'the', 'a' and 'an' in strings for normalization.
  def self.move_articles(str, position = :front, capitalize: true)
    return str unless [:front, :back, :none].include?(position)
    %w(the a an).each do |a|
      starts = str.downcase.start_with?(a + ' ')
      ends = str.downcase.end_with?(' ' + a)
      if starts && position != :front
        if position == :none
          str = str[(a.length + 1)..str.length]
        elsif position == :back
          str = str[(a.length + 1)..str.length] + (!ends ? ", #{capitalize ? a.capitalize : a}" : '')
        end
      end
      next unless ends && position != :back
      if position == :none
        str = str[0..-(a.length + 2)]
      elsif position == :front
        str = (!starts ? "#{capitalize ? a.capitalize : a} " : '') + str[0..-(a.length + 2)]
      end
    end
    str = str.strip.chop while str.strip.end_with?(',')
    str
  end
end

class String
  # Multi-split. Similar to split, but can be passed an array of delimiters to split on.
  def msplit(*delims)
    ary = [self]
    return ary if delims.empty?
    delims.flatten.each do |d|
      ary = ary.flat_map { |a| a.split d }
    end
    ary
  end

  # Split on delimiters
  def quote_split(*delimiters)
    encap_split('"\'', *delimiters)
  end

  alias qsplit quote_split

  # Split on only delimiters not between specific encapsulators
  # Various characters are special and automatically recognized such as parens
  # which automatically match anything between a begin and end character.
  def encap_split(encapsulator, *delimiters)
    pattern = case encapsulator
              when '('
                '\\(\\)'
              when '['
                '\\[\\]'
              when '{'<commit_msg>Added chars_uo_to method and pluralize.<commit_after>
# frozen_string_literal: true
require_relative 'matching'
require_relative 'roman'
require_relative 'fuzzy_matcher'
require_relative 'cases'
require_relative 'regexp'

module BBLib
  # Quickly remove any symbols from a string leaving only alpha-numeric characters and white space.
  def self.drop_symbols(str)
    str.gsub(/[^\w\s\d]|_/, '')
  end

  # Extract all integers from a string. Use extract_floats if numbers may contain decimal places.
  def self.extract_integers(str, convert: true)
    BBLib.extract_numbers(str, convert: false).reject { |r| r.include?('.') }
         .map { |m| convert ? m.to_i : m }
  end

  # Extracts all integers or decimals from a string into an array.
  def self.extract_floats(str, convert: true)
    BBLib.extract_numbers(str, convert: false).reject { |r| !r.include?('.') }
         .map { |m| convert ? m.to_f : m }
  end

  # Extracts any correctly formed integers or floats from a string
  def self.extract_numbers(str, convert: true)
    str.scan(/\d+\.\d+(?<=[^\.])|\d+(?<=[^\.])|\d+\.\d+$|\d+$/)
       .map { |f| convert ? (f.include?('.') ? f.to_f : f.to_i) : f }
  end

  # Used to move the position of the articles 'the', 'a' and 'an' in strings for normalization.
  def self.move_articles(str, position = :front, capitalize: true)
    return str unless [:front, :back, :none].include?(position)
    %w(the a an).each do |a|
      starts = str.downcase.start_with?(a + ' ')
      ends = str.downcase.end_with?(' ' + a)
      if starts && position != :front
        if position == :none
          str = str[(a.length + 1)..str.length]
        elsif position == :back
          str = str[(a.length + 1)..str.length] + (!ends ? ", #{capitalize ? a.capitalize : a}" : '')
        end
      end
      next unless ends && position != :back
      if position == :none
        str = str[0..-(a.length + 2)]
      elsif position == :front
        str = (!starts ? "#{capitalize ? a.capitalize : a} " : '') + str[0..-(a.length + 2)]
      end
    end
    str = str.strip.chop while str.strip.end_with?(',')
    str
  end

  # Displays a portion of an object (as a string) with an ellipse displayed
  # if the string is over a certain size.
  # Supported styles:
  # => front - "for exam..."
  # => back - "... example"
  # => middle - "... exam..."
  # => outter - "for e...ple"
  # The length of the too_long string is NOT factored into the cap
  def self.chars_up_to(str, cap, too_long = '...', style: :front)
    return str if str.to_s.size <= cap
    case style
    when :back
      "#{too_long}#{str[(str.size - cap)..-1]}"
    when :outter
      "#{str[0...(cap / 2).to_i + (cap.odd? ? 1 : 0)]}#{too_long}#{str[-(cap / 2).to_i..-1]}"
    when :middle
      "#{too_long}#{str[(str.size / 2 - cap / 2 - (cap.odd? ? 1 : 0)).to_i...(str.size / 2 + cap / 2).to_i]}#{too_long}"
    else
      "#{str[0...cap]}#{too_long}"
    end
  end

  def self.pluralize(num, base, plural = 's', singular = nil)
    num == 1 ? "#{base}#{singular}" : "#{base}#{plural}"
  end
end

class String
  # Multi-split. Similar to split, but can be passed an array of delimiters to split on.
  def msplit(*delims)
    ary = [self]
    return ary if delims.empty?
    delims.flatten.each do |d|
      ary = ary.flat_map { |a| a.split d }
    end
    ary
  end

  # Split on delimiters
  def quote_split(*delimiters)
    encap_split('"\'', *delimiters)
  end

  alias qsplit quote_split

  # Split on only delimiters not between specific encapsulators
  # Various characters are special and automatically recognized such as parens
  # which automatically match anything between a begin and end character.
  def encap_split(encapsulator, *delimiters)
    pattern = case encapsulator
              when '('
                '\\(\\)'
              when '['
                '\\[\\]'
              when '{'
{'fraction_non_alphanumeric': 0.14897386682415473, 'fraction_numerical': 0.0039864166543629115, 'mean_word_length': 2.5955414012738856, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2845376', 'n_tokens_mistral': 2347, 'n_tokens_neox': 2258, 'n_words': 672}