<commit_before># Author <NAME> (emonti at matasano)
require "base64"
require "stringio"
require 'zlib'
require 'open3'
require 'sha1'

module Rbkb
  DEFAULT_BYTE_ORDER=:big
  HEXCHARS = [("0".."9").to_a, ("a".."f").to_a].flatten
end

# Generates a "universally unique identifier"
def uuid
  (SHA1::sha1(rand.to_s)).to_s
end

# Generates a random alphanumeric string of 'size' bytes (8 by default)
def random_string(size = 8)
  chars = ('A'..'Z').to_a + ('a'..'z').to_a + ('0'..'9').to_a
  (1..size).collect{|a| chars[rand(chars.size)]}.join
end


# Simple syntactic sugar to pass any object to a block
def with(x)
  yield x if block_given?; x
end if not defined? with


#-----------------------------------------------------------------------------

# Mixins and class-specific items

class String
  # shortcut for hex sanity with regex
  def ishex? ; (self =~ /^[a-f0-9]+$/i)? true : false ; end 

  # Encode into percent-hexify url encoding format
  def urlenc(opts={})
    s=self
    plus = opts[:plus]
    unless (opts[:rx] ||= /[^A-Za-z0-9_\.~-]/).kind_of? Regexp
      raise "rx must be a regular expression for a character class"
    end
    hx = Rbkb::HEXCHARS

    s.gsub(opts[:rx]) do |c| 
      c=c[0]
      (plus and c==32)? '+' : "%" + (hx[(c >> 4)] + hx[(c & 0xf )])
    end
  end
  
  # Undo percent-hexified url encoding data
  def urldec(opts={})
    s=self
    s.gsub!('+', ' ') unless opts[:noplus]
    s.gsub(/%([A-Fa-f0-9]{2})/) {$1.hex.chr}
  end

  # Base64 encode
  def b64; Base64.encode64(self).tr("\n", ""); end

  # Base64 decode
  def d64; Base64.decode64(self); end

  # right-align to 'a' alignment padded with 'p'
  def ralign(a, p=' ')
    s=self
    p ||= ' '
    l = s.length
    pad = l.pad(a)
    s.rjust(pad+l, p)
  end

  # left-align to 'a' alignment padded with 'p'
  def lalign(a, p=' ')
    s=self
    p ||= ' '
    l = s.length
    pad = l.pad(a)
    s.ljust(pad+l, p)
  end


  # Convert a string to ASCII hex string. Supports a few options for format:
  #
  #   :delim - delimter between each hex byte
  #   :prefix - prefix before each hex byte
  #   :suffix - suffix after each hex byte
  # 
  def hexify(opts={})
    s=self<commit_msg>factored out Base64 lib<commit_after># Author <NAME> (emonti at matasano)
require "stringio"
require 'zlib'
require 'open3'
require 'sha1'

module Rbkb
  DEFAULT_BYTE_ORDER=:big
  HEXCHARS = [("0".."9").to_a, ("a".."f").to_a].flatten
end

# Generates a "universally unique identifier"
def uuid
  (SHA1::sha1(rand.to_s)).to_s
end

# Generates a random alphanumeric string of 'size' bytes (8 by default)
def random_string(size = 8)
  chars = ('A'..'Z').to_a + ('a'..'z').to_a + ('0'..'9').to_a
  (1..size).collect{|a| chars[rand(chars.size)]}.join
end


# Simple syntactic sugar to pass any object to a block
def with(x)
  yield x if block_given?; x
end if not defined? with


#-----------------------------------------------------------------------------

# Mixins and class-specific items

class String
  # shortcut for hex sanity with regex
  def ishex? ; (self =~ /^[a-f0-9]+$/i)? true : false ; end 

  # Encode into percent-hexify url encoding format
  def urlenc(opts={})
    s=self
    plus = opts[:plus]
    unless (opts[:rx] ||= /[^A-Za-z0-9_\.~-]/).kind_of? Regexp
      raise "rx must be a regular expression for a character class"
    end
    hx = Rbkb::HEXCHARS

    s.gsub(opts[:rx]) do |c| 
      c=c[0]
      (plus and c==32)? '+' : "%" + (hx[(c >> 4)] + hx[(c & 0xf )])
    end
  end
  
  # Undo percent-hexified url encoding data
  def urldec(opts={})
    s=self
    s.gsub!('+', ' ') unless opts[:noplus]
    s.gsub(/%([A-Fa-f0-9]{2})/) {$1.hex.chr}
  end

  # Base64 encode
  def b64(len=nil)
    ret = [self].pack("m").gsub("\n", "")
    if len and Numeric === len 
      ret.scan(/.{1,#{len}}/).join("\n") + "\n"
    else
      ret
    end
  end

  # Base64 decode
  def d64;  self.unpack("m")[0];  end

  # right-align to 'a' alignment padded with 'p'
  def ralign(a, p=' ')
    s=self
    p ||= ' '
    l = s.length
    pad = l.pad(a)
    s.rjust(pad+l, p)
  end

  # left-align to 'a' alignment padded with 'p'
  def lalign(a, p=' ')
    s=self
    p ||= ' '
    l = s.length
    pad = l.pad(a)
    s.ljust(pad+l, p)
  end


  # Convert a string to ASCII hex string. Supports a few options for format:
  #
  #   :delim - delimter between each hex byte
  #   :prefix - prefix before each hex byte
  #   :suffix - suffix after each hex byte
  # 
  def hexify(opts={})
    s=self
{'fraction_non_alphanumeric': 0.19870651204281892, 'fraction_numerical': 0.017395182872435324, 'mean_word_length': 2.4394171779141103, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6772007', 'n_tokens_mistral': 1788, 'n_tokens_neox': 1712, 'n_words': 479}