<commit_before>package main

import (
	"bytes"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"testing"
)

var (
	testInputFile     = "./pb/test.pb.go"
	testInputFileTemp = "./pb/test.pb.go_tmp"
)

func TestTagFromComment(t *testing.T) {
	var tests = []struct {
		comment string
		tag     string
	}{
		{comment: `//@inject_tag: valid:"abc"`, tag: `valid:"abc"`},
		{comment: `//   @inject_tag: valid:"abcd"`, tag: `valid:"abcd"`},
		{comment: `// @inject_tag:      valid:"xyz"`, tag: `valid:"xyz"`},
		{comment: `// fdsafsa`, tag: ""},
		{comment: `//@inject_tag:`, tag: ""},
		{comment: `// @inject_tag: json:"abc" yaml:"abc`, tag: `json:"abc" yaml:"abc`},
		{comment: `// test @inject_tag: json:"abc" yaml:"abc`, tag: `json:"abc" yaml:"abc`},
	}
	for _, test := range tests {
		result := tagFromComment(test.comment)
		if result != test.tag {
			t.Errorf("expected tag: %q, got: %q", test.tag, result)
		}
	}
}

func TestParseWriteFile(t *testing.T) {
	expectedTag := `valid:"ip" yaml:"ip" json:"overrided"`

	areas, err := parseFile(testInputFile, []string{})
	if err != nil {
		t.Fatal(err)
	}
	if len(areas) != 8 {
		t.Fatalf("expected 8 areas to replace, got: %d", len(areas))
	}
	area := areas[0]
	t.Logf("area: %v", area)
	if area.InjectTag != expectedTag {
		t.Errorf("expected tag: %q, got: %q", expectedTag, area.InjectTag)
	}

	// make a copy of test file
	contents, err := ioutil.ReadFile(testInputFile)
	if err != nil {
		t.Fatal(err)
	}
	if err = ioutil.WriteFile(testInputFileTemp, contents, 0644); err != nil {
		t.Fatal(err)
	}
	defer os.Remove(testInputFileTemp)

	if err = writeFile(testInputFileTemp, areas); err != nil {
		t.Fatal(err)
	}

	// check if file contains custom tag
	contents, err = ioutil.ReadFile(testInputFileTemp)
	if err != nil {
		t.Fatal(err)
	}
	expectedExpr := "Address[ \t]+string[ \t]+`protobuf:\"bytes,1,opt,name=Address,proto3\" json:\"overrided\" valid:\"ip\" yaml:\"ip\"`"
	matched, err := regexp.Match(expectedExpr, contents)
	if err != nil || matched != true {
		t.Error("file doesn't contains custom tag after writing")
		t.Log(string(contents))
	}
}

func TestNewTagItems(t *testing.T) {
	var tests = []struct {
		tag   string
		items tagItems
	}{
		{
			tag: `valid:"ip" yaml:"ip, required" json:"overrided"`,
			items: []tagItem{
				{key: "valid", value: `"ip"`},
				{key: "yaml", value: `"ip, required"`},
				{key: "json", value: `"overrided"`},
			},
		},
		{
			tag: `validate:"omitempty,oneof=a b c d"`,
			items: []tagItem{
				{key: "validate", value: `"omitempty,oneof=a b c d"`},
			},
		},
	}

	for _, test := range tests {
		for i, item := range newTagItems(test.tag) {
			if item.key != test.items[i].key || item.value != test.items[i].value {
				t.Errorf("wrong tag item for tag %s, expected %v, got: %v",
					test.tag, test.items[i], item)
			}
		}
	}
}

func TestContinueParsingWhenSkippingFields(t *testing.T) {
	expectedTags := []string{
		`valid:"ip" yaml:"ip" json:"overrided"`,
		`valid:"http|https"`,
		`valid:"nonzero"`,
		`validate:"omitempty"`,
		`xml:"-"`,
		`validate:"omitempty"`,
		`tag:"foo_bar"`,
		`tag:"foo"`,
		`tag:"bar"`,
	}

	areas, err := parseFile(testInputFile, []string{"xml"})
	if err != nil {
		t.Fatal(err)
	}

	if len(areas) != len(expectedTags) {
		t.Fatalf("expected %d areas to replace, got: %d", len(expectedTags), len(areas))
	}

	for i, a := range areas {
		if a.InjectTag != expectedTags[i] {
			t.Errorf("expected tag: %q, got: %q", expectedTags[i], a.InjectTag)
		}
	}
<commit_msg>correct tests based off new field entries<commit_after>package main

import (
	"bytes"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"testing"
)

var (
	testInputFile     = "./pb/test.pb.go"
	testInputFileTemp = "./pb/test.pb.go_tmp"
)

func TestTagFromComment(t *testing.T) {
	var tests = []struct {
		comment string
		tag     string
	}{
		{comment: `//@inject_tag: valid:"abc"`, tag: `valid:"abc"`},
		{comment: `//   @inject_tag: valid:"abcd"`, tag: `valid:"abcd"`},
		{comment: `// @inject_tag:      valid:"xyz"`, tag: `valid:"xyz"`},
		{comment: `// fdsafsa`, tag: ""},
		{comment: `//@inject_tag:`, tag: ""},
		{comment: `// @inject_tag: json:"abc" yaml:"abc`, tag: `json:"abc" yaml:"abc`},
		{comment: `// test @inject_tag: json:"abc" yaml:"abc`, tag: `json:"abc" yaml:"abc`},
	}
	for _, test := range tests {
		result := tagFromComment(test.comment)
		if result != test.tag {
			t.Errorf("expected tag: %q, got: %q", test.tag, result)
		}
	}
}

func TestParseWriteFile(t *testing.T) {
	expectedTag := `valid:"ip" yaml:"ip" json:"overrided"`

	areas, err := parseFile(testInputFile, []string{})
	if err != nil {
		t.Fatal(err)
	}
	if len(areas) != 9 {
		t.Fatalf("expected 9 areas to replace, got: %d", len(areas))
	}
	area := areas[0]
	t.Logf("area: %v", area)
	if area.InjectTag != expectedTag {
		t.Errorf("expected tag: %q, got: %q", expectedTag, area.InjectTag)
	}

	// make a copy of test file
	contents, err := ioutil.ReadFile(testInputFile)
	if err != nil {
		t.Fatal(err)
	}
	if err = ioutil.WriteFile(testInputFileTemp, contents, 0644); err != nil {
		t.Fatal(err)
	}
	defer os.Remove(testInputFileTemp)

	if err = writeFile(testInputFileTemp, areas); err != nil {
		t.Fatal(err)
	}

	// check if file contains custom tag
	contents, err = ioutil.ReadFile(testInputFileTemp)
	if err != nil {
		t.Fatal(err)
	}
	expectedExpr := "Address[ \t]+string[ \t]+`protobuf:\"bytes,1,opt,name=Address,proto3\" json:\"overrided\" valid:\"ip\" yaml:\"ip\"`"
	matched, err := regexp.Match(expectedExpr, contents)
	if err != nil || matched != true {
		t.Error("file doesn't contains custom tag after writing")
		t.Log(string(contents))
	}
}

func TestNewTagItems(t *testing.T) {
	var tests = []struct {
		tag   string
		items tagItems
	}{
		{
			tag: `valid:"ip" yaml:"ip, required" json:"overrided"`,
			items: []tagItem{
				{key: "valid", value: `"ip"`},
				{key: "yaml", value: `"ip, required"`},
				{key: "json", value: `"overrided"`},
			},
		},
		{
			tag: `validate:"omitempty,oneof=a b c d"`,
			items: []tagItem{
				{key: "validate", value: `"omitempty,oneof=a b c d"`},
			},
		},
	}

	for _, test := range tests {
		for i, item := range newTagItems(test.tag) {
			if item.key != test.items[i].key || item.value != test.items[i].value {
				t.Errorf("wrong tag item for tag %s, expected %v, got: %v",
					test.tag, test.items[i], item)
			}
		}
	}
}

func TestContinueParsingWhenSkippingFields(t *testing.T) {
	expectedTags := []string{
		`valid:"ip" yaml:"ip" json:"overrided"`,
		`valid:"-"`,
		`valid:"http|https"`,
		`valid:"nonzero"`,
		`validate:"omitempty"`,
		`xml:"-"`,
		`validate:"omitempty"`,
		`tag:"foo_bar"`,
		`tag:"foo"`,
		`tag:"bar"`,
	}

	areas, err := parseFile(testInputFile, []string{"xml"})
	if err != nil {
		t.Fatal(err)
	}

	if len(areas) != len(expectedTags) {
		t.Fatalf("expected %d areas to replace, got: %d", len(expectedTags), len(areas))
	}

	for i, a := range areas {
		if a.InjectTag != expectedTags[i] {
			t.Errorf("expected tag: %q, got: %q", expectedTags[i], a.InjectTag)
		}
	}

{'fraction_non_alphanumeric': 0.2044514196033671, 'fraction_numerical': 0.0025681266942502497, 'mean_word_length': 3.857934857934858, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4685897', 'n_tokens_mistral': 2954, 'n_tokens_neox': 2735, 'n_words': 619}