<commit_before>package sgf

import (
	"sort"
	"strings"
	"encoding/json"
)

type GameInfo map[string]string

func (gi GameInfo) SortedKeys() []string {
	var keys sort.StringSlice
	for k, _ := range gi {
		keys = append(keys, k)
	}
	sort.Sort(keys)
	return keys
}

func (gi GameInfo) String() string {
	str := ""
	for _, k := range gi.SortedKeys() {
		str += k + "[" + gi[k] + "]"
	}
	return ";" + str
}

func (gi GameInfo) ToJson() ([]byte, error) {
	json_map := gi.clone()
	for _, k := range gi.SortedKeys() {
		key := propsToKeys[strings.ToUpper(k)]
		if key != "" {
			json_map[key] = json_map[k]
			delete(json_map, k)
		}
	}
	return json.Marshal(json_map)
}

func (gi GameInfo) FromJson(json_str string) (GameInfo, error) {
	json_map, err := mapFromJson(json_str)
	if err != nil {
		return nil, err
	}

	gameInfo := make(GameInfo)

	for k, _ := range json_map {
		key := keysToProps[k]
		if key != "" {
			gameInfo[strings.ToUpper(key)] = json_map[k]
		} else {
			gameInfo[strings.ToUpper(k)] = json_map[k]
		}
	}
	return gameInfo, nil
}

func mapFromJson(json_str string) (map[string]string, error) {
	r := strings.NewReader(json_str)
	var json_map map[string]string
	err := json.NewDecoder(r).Decode(&json_map)
	if err != nil {
		return nil, err
	}

	return json_map, nil
}

func (gi GameInfo) clone() map[string]string {
	duplicate := make(map[string]string)
	for k, v := range gi {
		duplicate[k] = v
	}
	return duplicate
}

var propsToKeys = map[string]string{
	"AN": "Annotator",
	"PB": "BlackPlayerName",
	"BR": "BlackPlayerRank",
	"BT": "BlackPlayerTeam",
	"SZ": "Boardsize",
	"CA": "Charset",
	"C":  "Comment",
	"CP": "Copyright",
	"DT": "Date",
	"EV": "Event",
	"GC": "GameComment",
	"GN": "GameName",
	"HA": "Handicap",
	"KM": "Komi",
	"ON": "Opening",
	"OT": "Overtime",
	"PC": "Place",
	"RE": "Result",
	"RO": "Round",
	"RU": "Rules",
	"SO": "Source",
	"TM": "TimeLimits",
	"US": "User",
	"PW": "WhitePlayerName",
	"WR": "WhitePlayerRank",
	"WT": "WhitePlayerTeam",
}

var keysToProps = map[string]string{
	"Annotator":       "AN",
	"BlackPlayerName": "PB",
	"BlackPlayerRank": "BR",
	"BlackPlayerTeam": "BT",
	"Boardsize":       "SZ",
	"Charset":         "CA",
	"Comment":         "C",
	"Copyright":       "CP",
	"Date":            "DT",
	"Event":           "EV",
	"GameComment":     "GC",
	"GameName":        "GN",
	"Handicap":        "HA",
	"Komi":            "KM",
	"Opening":         "ON",
	"Overtime":        "OT",
	"Place":           "PC",
	"Result":          "RE",
	"Round":           "RO",
	"Rules":           "RU",
	"Source":          "SO",
	"TimeLimits":      "TM",
	"User":            "US",
	"WhitePlayerName": "PW",
	"WhitePlayerRank": "WR",
	"WhitePlayerTeam": "WT",
}<commit_msg>GameInfo helper fns: property2key(), key2property()<commit_after>package sgf

import (
	"sort"
	"strings"
	"encoding/json"
)

type GameInfo map[string]string

func (gi GameInfo) SortedKeys() []string {
	var keys sort.StringSlice
	for k, _ := range gi {
		keys = append(keys, k)
	}
	sort.Sort(keys)
	return keys
}

func (gi GameInfo) String() string {
	str := ""
	for _, k := range gi.SortedKeys() {
		str += k + "[" + gi[k] + "]"
	}
	return ";" + str
}

func (gi GameInfo) ToJson() ([]byte, error) {
	json_map := gi.clone()
	for _, k := range gi.SortedKeys() {
		key := property2key(k)
		if key != "" {
			json_map[key] = json_map[k]
			delete(json_map, k)
		}
	}
	return json.Marshal(json_map)
}

func (gi GameInfo) FromJson(json_str string) (GameInfo, error) {
	json_map, err := mapFromJson(json_str)
	if err != nil {
		return nil, err
	}

	gameInfo := make(GameInfo)

	for k, _ := range json_map {
		key := key2property(k)
		if key != "" {
			gameInfo[strings.ToUpper(key)] = json_map[k]
		} else {
			gameInfo[strings.ToUpper(k)] = json_map[k]
		}
	}
	return gameInfo, nil
}

func mapFromJson(json_str string) (map[string]string, error) {
	r := strings.NewReader(json_str)
	var json_map map[string]string
	err := json.NewDecoder(r).Decode(&json_map)
	if err != nil {
		return nil, err
	}

	return json_map, nil
}

func (gi GameInfo) clone() map[string]string {
	duplicate := make(map[string]string)
	for k, v := range gi {
		duplicate[k] = v
	}
	return duplicate
}

func property2key(propname string) string {
	return propsToKeys[strings.ToUpper(propname)]
}

var propsToKeys = map[string]string{
	"AN": "Annotator",
	"PB": "BlackPlayerName",
	"BR": "BlackPlayerRank",
	"BT": "BlackPlayerTeam",
	"SZ": "Boardsize",
	"CA": "Charset",
	"C":  "Comment",
	"CP": "Copyright",
	"DT": "Date",
	"EV": "Event",
	"GC": "GameComment",
	"GN": "GameName",
	"HA": "Handicap",
	"KM": "Komi",
	"ON": "Opening",
	"OT": "Overtime",
	"PC": "Place",
	"RE": "Result",
	"RO": "Round",
	"RU": "Rules",
	"SO": "Source",
	"TM": "TimeLimits",
	"US": "User",
	"PW": "WhitePlayerName",
	"WR": "WhitePlayerRank",
	"WT": "WhitePlayerTeam",
}

func key2property(key string) string {
	return keysToProps[strings.ToLower(key)]
}

var keysToProps = map[string]string{
	"annotator":       "AN",
	"blackplayername": "PB",
	"blackplayerrank": "BR",
	"blackplayerteam": "BT",
	"boardsize":       "SZ",
	"charset":         "CA",
	"comment":         "C",
	"copyright":       "CP",
	"date":            "DT",
	"event":           "EV",
	"gamecomment":     "GC",
	"gamename":        "GN",
	"handicap":        "HA",
	"komi":            "KM",
	"opening":         "ON",
	"overtime":        "OT",
	"place":           "PC",
	"result":          "RE",
	"round":           "RO",
	"rules":           "RU",
	"source":          "SO",
	"timelimits":      "TM",
	"user":            "US",
	"whiteplayername": "PW",
	"whiteplayerrank": "WR",
	"whiteplayerteam": "WT",
}
{'fraction_non_alphanumeric': 0.1992180558023814, 'fraction_numerical': 0.0010662875422072153, 'mean_word_length': 3.369565217391304, 'pattern_counts': {'":': 104, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1405408', 'n_tokens_mistral': 2308, 'n_tokens_neox': 2243, 'n_words': 506}