<commit_before><commit_msg>Faster decoding of document objects<commit_after>package gosearch

import (
	"encoding/json"
)

type Document struct {
	Index   string                 `json:"_index"`
	Type    string                 `json:"_type"`
	Id      string                 `json:"_id"`
	Version float64                `json:"_version,omitempty"`
	Exists  bool                   `json:"exists,omitempty"`
	Source  map[string]interface{} `json:"_source,omitempty"`
}

func (self *Document) UnmarshalJSON(data []byte) error {

	var objmap map[string]interface{}

	if err := json.Unmarshal(data, &objmap); err != nil {
		return err
	}

	if _, ok := objmap["fields"]; ok {
		objmap["_source"] = objmap["fields"]
		delete(objmap, "fields")
	}

	if prop, ok := objmap["_type"]; ok {
		self.Type = prop.(string)
	}

	if prop, ok := objmap["_id"]; ok {
		self.Id = prop.(string)
	}
	if prop, ok := objmap["_version"]; ok {
		self.Version = prop.(float64)
	}
	if prop, ok := objmap["exists"]; ok {
		self.Exists = prop.(bool)
	}

	if prop, ok := objmap["_source"]; ok {
		self.Source = prop.(map[string]interface{})
	}

	return nil
}
{'fraction_non_alphanumeric': 0.1585040071237756, 'fraction_numerical': 0.003561887800534283, 'mean_word_length': 3.0142857142857142, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3159111', 'n_tokens_mistral': 393, 'n_tokens_neox': 370, 'n_words': 91}