<commit_before>package google

import (
	"reflect"
	"testing"
)

func TestAddPublicKey(t *testing.T) {
	tests := []struct {
		Name      string
		Metadata  map[string]interface{}
		User      string
		PublicKey string
		Expected  map[string]interface{}
	}{
		{
			Name: `no user SSH keys`,
			Metadata: map[string]interface{}{
				`unrelated`: 5,
			},
			User:      `user`,
			PublicKey: `pk`,
			Expected: map[string]interface{}{
				`ssh-keys`:  `user:pk`,
				`unrelated`: 5,
			},
		},
		{
			Name: `user SSH keys new format`,
			Metadata: map[string]interface{}{
				`ssh-keys`: `user_custom:pk_custom`,
			},
			User:      `user`,
			PublicKey: `pk`,
			Expected: map[string]interface{}{
				`ssh-keys`: `user:pk\nuser_custom:pk_custom`,
			},
		},
		{
			Name: `user SSH keys deprecated format`,
			Metadata: map[string]interface{}{
				`sshKeys`: `user_custom:pk_custom`,
			},
			User:      `user`,
			PublicKey: `pk`,
			Expected: map[string]interface{}{
				`sshKeys`: `user:pk\nuser_custom:pk_custom`,
			},
		},
		{
			Name: `user SSH keys both formats`,
			Metadata: map[string]interface{}{
				`ssh-keys`: `user_custom_new:pk_custom_new`,
				`sshKeys`:  `user_custom_old:pk_custom_old`,
			},
			User:      `user`,
			PublicKey: `pk`,
			Expected: map[string]interface{}{
				`ssh-keys`: `user:pk\nuser_custom_new:pk_custom_new`,
				`sshKeys`:  `user_custom_old:pk_custom_old`,
			},
		},
	}

	for _, test := range tests {
		t.Run(test.Name, func(t *testing.T) {
			metadata := addPublicKey(test.Metadata, test.User, test.PublicKey)
			if !reflect.DeepEqual(metadata, test.Expected) {
				t.Fatalf(`want metadata = %#v; got %#v`, test.Expected, metadata)
			}
		})
	}
}<commit_msg>provider/google: add unit tests for flatten function<commit_after>package google

import (
	"reflect"
	"testing"
)

func TestAddPublicKey(t *testing.T) {
	tests := map[string]struct {
		Name      string
		Metadata  map[string]interface{}
		User      string
		PublicKey string
		Expected  map[string]interface{}
	}{
		"no user SSH keys": {
			Metadata: map[string]interface{}{
				"unrelated": 5,
			},
			User:      "user",
			PublicKey: "pk",
			Expected: map[string]interface{}{
				"ssh-keys":  "user:pk",
				"unrelated": 5,
			},
		},
		"user SSH keys new format": {
			Metadata: map[string]interface{}{
				"ssh-keys": "user_custom:pk_custom",
			},
			User:      "user",
			PublicKey: "pk",
			Expected: map[string]interface{}{
				"ssh-keys": "user:pk\\nuser_custom:pk_custom",
			},
		},
		"user SSH keys deprecated format": {
			Metadata: map[string]interface{}{
				"sshKeys": "user_custom:pk_custom",
			},
			User:      "user",
			PublicKey: "pk",
			Expected: map[string]interface{}{
				"sshKeys": "user:pk\\nuser_custom:pk_custom",
			},
		},
		"user SSH keys both formats": {
			Metadata: map[string]interface{}{
				"ssh-keys": "user_custom_new:pk_custom_new",
				"sshKeys":  "user_custom_old:pk_custom_old",
			},
			User:      "user",
			PublicKey: "pk",
			Expected: map[string]interface{}{
				"ssh-keys": "user:pk\\nuser_custom_new:pk_custom_new",
				"sshKeys":  "user_custom_old:pk_custom_old",
			},
		},
	}

	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			metadata := addPublicKey(test.Metadata, test.User, test.PublicKey)
			if !reflect.DeepEqual(metadata, test.Expected) {
				t.Fatalf("want metadata = %#v; got %#v", test.Expected, metadata)
			}
		})
	}
}

func TestFlatten(t *testing.T) {
	tests := map[string]struct {
		Data     []map[string]interface{}
		Expected map[string]interface{}
	}{
		"nil data": {
			Data:     nil,
			Expected: map[string]interface{}{},
		},
		"nil map": {
			Data: []map[string]interface{}{
				nil,
				{
					"user-data": "data",
				},
			},
			Expected: map[string]interface{}{
				"user-data": "data",
			},
		},
		"single map": {
			Data: []map[string]interface{}{
				{
					"user-data": "data",
				},
			},
			Expected: map[string]interface{}{
				"user-data": "data",
			},
		},
		"redefined map fields": {
			Data: []map[string]interface{}{
				{
					"user-data": "first",
				},
				{
					"user-data": "second",
				},
			},
			Expected: map[string]interface{}{
				"user-data": "first\nsecond",
			},
		},
		"different map fields": {
			Data: []map[string]interface{}{
				{
					"aaaa": "first",
				},
				{
					"bbbb": "second",
				},
			},
			Expected: map[string]interface{}{
				"aaaa": "first",
				"bbbb": "second",
			},
		},
		"map fields mix": {
			Data: []map[string]interface{}{
				{
					"aaaa": "first",
				},
				{
					"bbbb": "second",
				},
				{
					"aaaa": "A",
					"bbbb": "B",
					"cccc": "C",
				},
			},
			Expected: map[string]interface{}{
				"aaaa": "first\nA",
				"bbbb": "second\nB",
				"cccc": "C",
			},
		},
	}

	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			flattened := flatten(test.Data)
			if !reflect.DeepEqual(flattened, test.Expected) {
				t.Fatalf("want flattened = %#v; got %#v", test.Expected, flattened)
			}
		})
	}
}
{'fraction_non_alphanumeric': 0.18454763811048838, 'fraction_numerical': 0.0008006405124099279, 'mean_word_length': 3.046153846153846, 'pattern_counts': {'":': 40, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3951766', 'n_tokens_mistral': 2278, 'n_tokens_neox': 1816, 'n_words': 346}