<commit_before>
require 'rubygems'
require 'bundler/setup'

require 'toml'
require 'minitest/autorun'

class TestGenerator < MiniTest::Test
  def setup
    @doc = {
      "integer" => 1,
      "float" => 3.14159,
      "true" => true,
      "false" => false,
      "string" => "hi",
      "array" => [[1], [2], [3]],
      "key" => {
        "group" => {
          "value" => "lol"
        }
      },
      "date" => DateTime.now
    }
    
  end
  
  def test_generator
    doc = @doc.clone
    body = TOML::Generator.new(doc).body

    doc_parsed = TOML::Parser.new(body).parsed
    
    # Extracting dates since Ruby's DateTime equality testing sucks.
    original_date = doc.delete "date"
    parsed_date = doc_parsed.delete "date"
    
    assert_equal doc, doc_parsed
    assert_equal original_date.to_time.to_s, parsed_date.to_time.to_s
  end
end<commit_msg>generator: add test for nil key value<commit_after>
require 'rubygems'
require 'bundler/setup'

require 'toml'
require 'minitest/autorun'

class TestGenerator < MiniTest::Test
  def setup
    @doc = {
      "integer" => 1,
      "float" => 3.14159,
      "true" => true,
      "false" => false,
      "string" => "hi",
      "array" => [[1], [2], [3]],
      "key" => {
        "group" => {
          "value" => "lol"
        },
        "nil_table" => {}
      },
      "date" => DateTime.now,
      "nil" => nil
    }
    
  end
  
  def test_generator
    doc = @doc.clone
    body = TOML::Generator.new(doc).body

    doc_parsed = TOML::Parser.new(body).parsed
    
    # Extracting dates since Ruby's DateTime equality testing sucks.
    original_date = doc.delete "date"
    parsed_date = doc_parsed.delete "date"
    
    # removing the nil value
    remove_nil = doc.delete "nil"
    
    assert_equal doc, doc_parsed
    assert_equal original_date.to_time.to_s, parsed_date.to_time.to_s
  end
end
{'fraction_non_alphanumeric': 0.13911182450508294, 'fraction_numerical': 0.010700909577314071, 'mean_word_length': 2.3512544802867383, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5751024', 'n_tokens_mistral': 660, 'n_tokens_neox': 659, 'n_words': 147}