<commit_before>require "openssl"

module Cfcrypto
  def self.hex2str(str)
    [str].pack("H*")
  end

  def self.str2hex(str)
    str.unpack("H*").first
  end

  def self.b64encode(bin)
    # encode without newlines
    [bin].pack("m0")
  end

  def self.b64decode(str)
    # decode with or without newlines
    str.unpack("m").first
  end

  # def self.hex_xor(buf1, buf2)
  #   # hex strings
  #   raise "Buffers must be same length" if buf1.length != buf2.length
  #   in1 = hex2str(buf1)
  #   in2 = hex2str(buf2)
  #   res = in1.bytes.zip(in2.bytes).map { |a, b| a ^ b }.pack("C*")
  #   str2hex(res)
  # end

  def self.xor(str1, str2)
    # byte strings
    raise "strings must be same length" if str1.length != str2.length
    str1.bytes.zip(str2.bytes).map { |a, b| a ^ b }.pack("C*")
  end

  def self.xor_key(str1, key)<commit_msg>tools: remove unused hex string xor function<commit_after>require "openssl"

module Cfcrypto
  def self.hex2str(str)
    [str].pack("H*")
  end

  def self.str2hex(str)
    str.unpack("H*").first
  end

  def self.b64encode(bin)
    # encode without newlines
    [bin].pack("m0")
  end

  def self.b64decode(str)
    # decode with or without newlines
    str.unpack("m").first
  end

  def self.xor(str1, str2)
    # byte strings
    raise "strings must be same length" if str1.length != str2.length
    str1.bytes.zip(str2.bytes).map { |a, b| a ^ b }.pack("C*")
  end

  def self.xor_key(str1, key)
{'fraction_non_alphanumeric': 0.1436502428868841, 'fraction_numerical': 0.028452463566967384, 'mean_word_length': 2.939890710382514, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5794232', 'n_tokens_mistral': 588, 'n_tokens_neox': 566, 'n_words': 136}