<commit_before>package main

import (
	"fmt"
	elastigo "github.com/barnslig/elastigo/lib"
	"log"
)

var (
	es_conn *elastigo.Conn
)

type FtpEntry struct {
	Url  string
	Path string
}

type FileEntry struct {
	Filename string
	Size     uint64
	Servers  []FtpEntry
}

type hash map[string]interface{}

func initElastics(host string) {
	es_conn = elastigo.NewConn()
	es_conn.Domain = host

	es_conn.CreateIndex("torture")

	// enable timestamps
	err := es_conn.PutMapping("torture", "file", FileEntry{}, elastigo.MappingOptions{
		Timestamp: elastigo.TimestampOptions{
			Enabled: true,
			Store:   true,
		},
		Properties: hash{
			"Filename": hash{
				"type":  "string",
				"index": "not_analyzed",
			},
		},
	})
	if err != nil {
		log.Fatal(err)
	}
}

func addToElastic(file FileEntry) {
	// look up the file by the name
	searchJson := fmt.Sprintf(`{
		"query": {
			"filtered": {
				"filter": {
					"term": {
						"Filename": "%s"
					}
				}
			}
		}
	}`, file.Filename)
	query_response, err := es_conn.Search("torture", "file", nil, searchJson)
	if err != nil {
		log.Print(err)
		return
	}
	if query_response.Hits.Len() > 0 {
		log.Println(query_response.Hits.Hits[0].Id)
		_, err := es_conn.Update("torture", "file", query_response.Hits.Hits[0].Id, nil, hash{
			"script": "ctx._source.Servers += Server",
			"params": hash{
				"Server": hash{
					"Url":  file.Servers[0].Url,
					"Path": file.Servers[0].Path,
				},
			},
		})
		if err != nil {
			log.Print(err)
		}
	} else {
		_, err := es_conn.Index("torture", "file", "", nil, file)
		if err != nil {
			log.Print(err)
		}
	}
}<commit_msg>Prevent double-entries and weird mergings<commit_after>package main

import (
	"fmt"
	elastigo "github.com/barnslig/elastigo/lib"
	"log"
)

var (
	es_conn *elastigo.Conn
)

type FtpEntry struct {
	Url  string
	Path string
}

type FileEntry struct {
	Filename string
	Size     uint64
	Servers  []FtpEntry
}

type hash map[string]interface{}

func initElastics(host string) {
	es_conn = elastigo.NewConn()
	es_conn.Domain = host

	es_conn.CreateIndex("torture")

	// enable timestamps
	err := es_conn.PutMapping("torture", "file", FileEntry{}, elastigo.MappingOptions{
		Timestamp: elastigo.TimestampOptions{
			Enabled: true,
			Store:   true,
		},
		Properties: hash{
			"Filename": hash{
				"type":  "string",
				"index": "not_analyzed",
			},
		},
	})
	if err != nil {
		log.Fatal(err)
	}
}

func addToElastic(file FileEntry) {
	// look up the file by the name
	searchJson := fmt.Sprintf(`{
		"query": {
			"filtered": {
				"filter": {
					"bool": {
						"must": [
							{"term": {
								"Filename": "%s"
							}},
							{"term": {
								"Size": %d
							}}
						]
					}
				}
			}
		}
	}`, file.Filename, file.Size)
	query_response, err := es_conn.Search("torture", "file", nil, searchJson)
	if err != nil {
		log.Print(err)
		return
	}
	if query_response.Hits.Len() > 0 {
		log.Println(query_response.Hits.Hits[0].Id)

		_, err := es_conn.Update("torture", "file", query_response.Hits.Hits[0].Id, nil, hash{
			"script": "ctx._source.Servers.contains(Server) ? (ctx.op = \"none\") : (ctx._source.Servers += Server)",
			"params": hash{
				"Server": hash{
					"Url":  file.Servers[0].Url,
					"Path": file.Servers[0].Path,
				},
			},
		})
		if err != nil {
			log.Print(err)
		}
	} else {
		_, err := es_conn.Index("torture", "file", "", nil, file)
		if err != nil {
			log.Print(err)
		}
	}
}
{'fraction_non_alphanumeric': 0.170411438576014, 'fraction_numerical': 0.004085205719288007, 'mean_word_length': 3.023474178403756, 'pattern_counts': {'":': 30, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4173892', 'n_tokens_mistral': 1547, 'n_tokens_neox': 1326, 'n_words': 267}