<commit_before>require "./helpers.rb"
require "./1-3_one_byte_xor_cipher.rb"
require "./1-5_repeat_key_xor.rb"

def break_repeat_key_xor(filename)
  data64 = File.open(filename).map(&:strip).join("")
  data = data64.unpack("m").first

  # The KEYSIZE with the smallest normalized edit distance is probably the key.
  # You could proceed perhaps with the smallest 2-3 KEYSIZE values. Or take 4
  # KEYSIZE blocks instead of 2 and average the distances.
  best_key_sizes = key_size_stats(data).first(20).map(&:first)
  puts "Following were the best key sizes found: #{best_key_sizes}"

  best_key_sizes.each do |key_size|
    puts "Key size: #{key_size}"
    blocks = transpose_blocks(data, key_size)

    # Solve each block as if it was single-character XOR. For each block, the
    # single-byte XOR key that produces the best looking histogram is the
    # repeating-key XOR key byte for that block. Put them together and you have
    # the key.
    key = blocks.map do |idx, block|
      best_result = single_byte_xor_cipher(encode_hex(block))[0]
      key_part = best_result[0]
      [idx, key_part]
    end.
      sort_by{ |i| i[0] }.
      map{ |i| i[1].chr }.
      join("")

    puts "Probable key: '#{key}'"
  end
end

def key_size_stats(data)
  # Let KEYSIZE be the guessed length of the key; try values from 2 to (say) 40.
  # For each KEYSIZE, take the first KEYSIZE worth of bytes, and the second
  # KEYSIZE worth of bytes, and find the edit distance between them. Normalize
  # this result by dividing by KEYSIZE.
  (2..40).each_with_object([]) do |key_size, a|
    chunk_1 = data[0..key_size-1]
    chunk_2 = data[key_size..2*key_size-1]

    a << [key_size, hamming_dist(chunk_1, chunk_2).to_f / key_size]
  end.sort_by{ |i| i[1] }
end

# Creates blocks of size 'size' and transposes them to create 'size' number
# of new blocks
# For example: transpose_blocks("abcdef", 2)
# => ["ace", "bdf"]
def transpose_blocks(data, size)
  blocks = Hash.new { |h, k| h[k] = [] }

  # Now that you probably know the KEYSIZE: break the ciphertext into blocks of
  # KEYSIZE length.
  data.split("").each_slice(size) do |slice|
    slice.each_with_index do |char, i|
    # Now transpose the blocks: make a block that is the first byte of every
    # block, and a block that is the second byte of every block, and so on.
      blocks[i] << char
    end
  end

  blocks.each do |k, v|
    blocks[k] = v.join("")
  end
end

# break_repeat_key_xor("./1-6_data.txt")

# Encrypts a data file with repeating key XOR and Base-64s it
# Example:
# encrypt_file("1-6_data_test.txt", "Hello world")
def encrypt_file(filename, key)
  File.open(filename) do |file|
    data = file.read
    encrypted64 = hex_to_base64(repeat_key_xor(data, key))

    File.open("encrypted64_" + filename, "w") do |out_file|<commit_msg>Fixed key_size_stats to give correct key at the top<commit_after>require "./helpers.rb"
require "./1-3_one_byte_xor_cipher.rb"
require "./1-5_repeat_key_xor.rb"

def break_repeat_key_xor(filename)
  data64 = File.open(filename).map(&:strip).join("")
  data = data64.unpack("m").first

  # The KEYSIZE with the smallest normalized edit distance is probably the key.
  # You could proceed perhaps with the smallest 2-3 KEYSIZE values. Or take 4
  # KEYSIZE blocks instead of 2 and average the distances.
  best_key_sizes = key_size_stats(data).first(5).map(&:first)
  puts "Following were the best key sizes found: #{best_key_sizes}"

  best_key_sizes.each do |key_size|
    puts "Key size: #{key_size}"
    blocks = transpose_blocks(data, key_size)

    # Solve each block as if it was single-character XOR. For each block, the
    # single-byte XOR key that produces the best looking histogram is the
    # repeating-key XOR key byte for that block. Put them together and you have
    # the key.
    key = blocks.map do |idx, block|
      best_result = single_byte_xor_cipher(encode_hex(block))[0]
      key_part = best_result[0]
      [idx, key_part]
    end.
      sort_by{ |i| i[0] }.
      map{ |i| i[1].chr }.
      join("")

    puts "Probable key: '#{key}'"
    puts "=========================================================="
    puts "Decrypted text: '#{decrypt_64file("./1-6_data.txt", key)}'"
    puts "=========================================================="
  end
end

def key_size_stats(data)
  # Let KEYSIZE be the guessed length of the key; try values from 2 to (say) 40.
  # For each KEYSIZE, take the first KEYSIZE worth of bytes, and the second
  # KEYSIZE worth of bytes, and find the edit distance between them. Normalize
  # this result by dividing by KEYSIZE.
  (2..40).each_with_object([]) do |key_size, a|
    dists = data.split("").
      each_slice(2 * key_size).
      with_index.
      select do |slice_arr, i|
        i < (data.length / (2 * key_size))
      end.
      map do |slice_arr, i|
      slice = slice_arr.join
      chunk_1 = slice[0..key_size-1]
      chunk_2 = slice[key_size..2*key_size-1]

      hamming_dist(chunk_1, chunk_2).to_f / key_size
    end

    avg_dist = dists.inject(&:+) / dists.count

    a << [key_size, avg_dist]
  end.sort_by{ |i| i[1] }
end

# Creates blocks of size 'size' and transposes them to create 'size' number
# of new blocks
# For example: transpose_blocks("abcdef", 2)
# => ["ace", "bdf"]
def transpose_blocks(data, size)
  blocks = Hash.new { |h, k| h[k] = [] }

  # Now that you probably know the KEYSIZE: break the ciphertext into blocks of
  # KEYSIZE length.
  data.split("").each_slice(size) do |slice|
    slice.each_with_index do |char, i|
    # Now transpose the blocks: make a block that is the first byte of every
    # block, and a block that is the second byte of every block, and so on.
      blocks[i] << char
    end
  end

  blocks.each do |k, v|
    blocks[k] = v.join("")
  end
end

# Encrypts a data file with repeating key XOR and Base-64s it
# Example:
# encrypt_file("1-6_data_test.txt", "Hello world")
def encrypt_file(filename, key)
  File.open(filename) do |file|
    data = file.read
    encrypted64 = hex_to_base64(repeat_key_xor(data, key))

    File.open("encrypted64_" + filename, "w") do |out_file|
{'fraction_non_alphanumeric': 0.13250449860952068, 'fraction_numerical': 0.015540651071486994, 'mean_word_length': 3.4595185995623634, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1331416', 'n_tokens_mistral': 2150, 'n_tokens_neox': 2066, 'n_words': 680}