<commit_before>package structure

import (
	"testing"
)

func TestNormalizeJsonString_valid(t *testing.T) {
	// Well formatted and valid.
	validJson := `{
   "abc": {
      "def": 123,
      "xyz": [
         {
            "a": "ホリネズミ"
         },
         {
            "b": "1\\n2"
         }
      ]
   }
}`
	expected := `{"abc":{"def":123,"xyz":[{"a":"ホリネズミ"},{"b":"1\\n2"}]}}`

	actual, err := NormalizeJsonString(validJson)
	if err != nil {
		t.Fatalf("Expected not to throw an error while parsing JSON, but got: %s", err)
	}

	if actual != expected {
		t.Fatalf("Got:\n\n%s\n\nExpected:\n\n%s\n", actual, expected)
	}<commit_msg>helper/structure: More cases for NormalizeJsonString tests<commit_after>package structure

import (
	"testing"
)

func TestNormalizeJsonString_valid(t *testing.T) {
	// Well formatted and valid.
	validJson := `{
   "abc": {
      "def": 123,
      "xyz": [
         {
            "a": "ホリネズミ"
         },
         {
            "b": "1\\n2"
         }
      ]
   }
}`
	expected := `{"abc":{"def":123,"xyz":[{"a":"ホリネズミ"},{"b":"1\\n2"}]}}`

	actual, err := NormalizeJsonString(validJson)
	if err != nil {
		t.Fatalf("Expected not to throw an error while parsing JSON, but got: %s", err)
	}

	if actual != expected {
		t.Fatalf("Got:\n\n%s\n\nExpected:\n\n%s\n", actual, expected)
	}

	// Well formatted but not valid,
	// missing closing square bracket.
	invalidJson := `{
   "abc": {
      "def": 123,
      "xyz": [
         {
            "a": "1"
         }
      }
   }
}`
	actual, err = NormalizeJsonString(invalidJson)
	if err == nil {
		t.Fatalf("Expected to throw an error while parsing JSON, but got: %s", err)
	}

	// We expect the invalid JSON to be shown back to us again.
	if actual != invalidJson {
		t.Fatalf("Got:\n\n%s\n\nExpected:\n\n%s\n", actual, invalidJson)
	}

	// Verify that it leaves strings alone
	testString := "2016-07-28t04:07:02z\nsomething else"
	expected = "2016-07-28t04:07:02z\nsomething else"
	actual, err = NormalizeJsonString(testString)
	if err == nil {
		t.Fatalf("Expected to throw an error while parsing JSON, but got: %s", err)
	}

	if actual != expected {
		t.Fatalf("Got:\n\n%s\n\nExpected:\n\n%s\n", actual, expected)
	}
{'fraction_non_alphanumeric': 0.1785552021808269, 'fraction_numerical': 0.023625624716038164, 'mean_word_length': 2.580487804878049, 'pattern_counts': {'":': 24, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7472852', 'n_tokens_mistral': 871, 'n_tokens_neox': 821, 'n_words': 187}