<commit_before># You'll need: <EMAIL>:lithium3141/HuffmanCoding.git
require './coder.rb'
# require 'file'

puts "This shouldn't be all that hard..."

# Load the file
filename = "./MCP4822-X_P.bxl" # 296091648
file = open(filename,"rb")

# Pull off the first 4 bytes as the length of the original file
original_length = file.read(4)
length_array = original_length.bytes.map { |n| n.to_s(16) }
puts "The first four bytes (in hex) are: #{length_array}"
puts "That's the original files length (in bytes): #{original_length.unpack('L>').first}" # L is 32 bit, > is Big endian


##===============
## Let's output the first four bytes in HEX just to make sure we're on the right track:
# og_hex_length = []
# original_length.each do |byte|
# 	og_hex_length << byte.to_s(16)
# end	
# puts "First four converted into hex: #{og_hex_length}"
##===============

##===============
## To check the bxl file format, let's try validating the size of the original file based on the EAGLE
## file output by the REAL bxl reader software:
# expected_file = "./2014-05-28_10-36-23_Library.scr"
# correct_file = File.new(expected_file,"r")
# whole_correct_file = correct_file.read.bytes.to_a
# puts "Expected file length: #{whole_correct_file.length}" # OOPS! This is 3301 which is way off.
## What this tells us is that the first four bytes of the bxl file is NOT the full length of the output
## but it is probably the full length of the decoded file that has CHUNKS of CAD files IN it!
##
## So, we need to just try reading the whole damn thing. Whatever.
##===============

# Read the rest of the file (We already seeked paste the first 4 bytes!)
whole_file = file.read.bytes.to_a
# puts "Whole file: #{whole_file}"
<commit_msg>Removing some really useless code and making some notes about what huffman library I was attempting to use.<commit_after># I took the coder.rb from: <EMAIL>:lithium3141/HuffmanCoding.git
#
# Thanks to lithium3141 for "letting me" borrow their code...
#
# Honestly, this probably isn't the right way to do Huffman decoding for bxl files.
# I read over on DangerousPrototypes in a comment that the first 4 bytes of the bxl
# file is the file length of the original file, before it was compressed into a bxl.
#
# I have no idea if this is true. I doubt this huffman decoding code will work
# correctly given the actual format of bxl files.
#
require './coder.rb'

# Load the file
filename = "./MCP4822-X_P.bxl" # original file length: 296091648
file = open(filename,"rb")

# Pull off the first 4 bytes as the length of the original file
original_length = file.read(4)
length_array = original_length.bytes.map { |n| n.to_s(16) }
puts "The first four bytes (in hex) are: #{length_array}"
puts "That's the original files length (in bytes): #{original_length.unpack('L>').first}" # L is 32 bit, > is Big endian
puts

# Read the rest of the file (We already seeked paste the first 4 bytes!)
whole_file = file.read.bytes.to_a
# puts "Whole file: #{whole_file}"

{'fraction_non_alphanumeric': 0.12086428089128967, 'fraction_numerical': 0.02464550979068197, 'mean_word_length': 4.466789667896679, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 1, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7171119', 'n_tokens_mistral': 1002, 'n_tokens_neox': 925, 'n_words': 398}