<commit_before>package elastic

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/juju/errors"
)

// Although there are many Elasticsearch clients with Go, I still want to implement one by myself.
// Because we only need some very simple usages.
type Client struct {
	Addr string
	User string
	Password string

	c *http.Client
}

type ClientConfig struct {
	Addr string
	User string
	Password string
}


func NewClient(conf *ClientConfig) *Client {
	c := new(Client)

	c.Addr = conf.Addr
	c.User = conf.User
	c.Password = conf.Password

	c.c = &http.Client{}

	return c
}

type ResponseItem struct {
	ID      string                 `json:"_id"`
	Index   string                 `json:"_index"`
	Type    string                 `json:"_type"`
	Version int                    `json:"_version"`
	Found   bool                   `json:"found"`
	Source  map[string]interface{} `json:"_source"`
}

type Response struct {
	Code int
	ResponseItem
}

// See http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/bulk.html
const (
	ActionCreate = "create"
	ActionUpdate = "update"
	ActionDelete = "delete"
	ActionIndex  = "index"
)

type BulkRequest struct {
	Action string
	Index  string
	Type   string
	ID     string
	Parent string
	JoinField string
	JoinFieldName string

	HardCrud bool

	Data map[string]interface{}
	DeleteFields map[string]interface{}

}

func (r *BulkRequest) bulk(buf *bytes.Buffer) error {
	meta := make(map[string]map[string]string)
	metaData := make(map[string]string)
	if len(r.Index) > 0 {
		metaData["_index"] = r.Index
	}
	if len(r.Type) > 0 {
		metaData["_type"] = r.Type
	}

	if len(r.ID) > 0 {
		metaData["_id"] = r.ID
	}
	if len(r.JoinField) > 0 {
		if len(r.Parent) > 0 {
			r.Data[r.JoinField] = map[string]interface{}{
				"name" : r.JoinFieldName,
				"parent" : r.Parent,
			}
			metaData["_routing"] = r.Parent
		} else {
			r.Data[r.JoinField] = map[string]interface{}{
				"name" : r.JoinFieldName,
			}
		}
	} else if len(r.Parent) > 0 {
		metaData["_parent"] = r.Parent
	}

	if r.HardCrud {
		meta[r.Action] = metaData
	} else {
		meta["update"] = metaData // all requests are update in this case
	}
	data, err := json.Marshal(meta)
	if err != nil {
		return errors.Trace(err)
	}
	if r.Action != ActionUpdate || (len(r.Data) > 1 || (len(r.JoinField) == 0 && len(r.Data) == 1)) {

		buf.Write(data)
		buf.WriteByte('\n')
	}

	switch r.Action {
	case ActionDelete:
		if !r.HardCrud {
			var del bytes.Buffer
			for k := range r.Data {
				del.WriteString("ctx._source.remove(\"")
				del.WriteString(k)
				del.WriteString("\");")
			}
			delScript := map[string]interface{}{
						"inline" : del.String(),
						"lang" : "painless",
					}<commit_msg># of retries on conflict<commit_after>package elastic

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/juju/errors"
)

// Although there are many Elasticsearch clients with Go, I still want to implement one by myself.
// Because we only need some very simple usages.
type Client struct {
	Addr string
	User string
	Password string

	c *http.Client
}

type ClientConfig struct {
	Addr string
	User string
	Password string
}


func NewClient(conf *ClientConfig) *Client {
	c := new(Client)

	c.Addr = conf.Addr
	c.User = conf.User
	c.Password = conf.Password

	c.c = &http.Client{}

	return c
}

type ResponseItem struct {
	ID      string                 `json:"_id"`
	Index   string                 `json:"_index"`
	Type    string                 `json:"_type"`
	Version int                    `json:"_version"`
	Found   bool                   `json:"found"`
	Source  map[string]interface{} `json:"_source"`
}

type Response struct {
	Code int
	ResponseItem
}

// See http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/bulk.html
const (
	ActionCreate = "create"
	ActionUpdate = "update"
	ActionDelete = "delete"
	ActionIndex  = "index"
)

type BulkRequest struct {
	Action string
	Index  string
	Type   string
	ID     string
	Parent string
	JoinField string
	JoinFieldName string

	HardCrud bool

	Data map[string]interface{}
	DeleteFields map[string]interface{}

}

func (r *BulkRequest) bulk(buf *bytes.Buffer) error {
	meta := make(map[string]map[string]string)
	metaData := make(map[string]string)
	if len(r.Index) > 0 {
		metaData["_index"] = r.Index
	}
	if len(r.Type) > 0 {
		metaData["_type"] = r.Type
	}

	if len(r.ID) > 0 {
		metaData["_id"] = r.ID
	}
	if len(r.JoinField) > 0 {
		if len(r.Parent) > 0 {
			r.Data[r.JoinField] = map[string]interface{}{
				"name" : r.JoinFieldName,
				"parent" : r.Parent,
			}
			metaData["_routing"] = r.Parent
		} else {
			r.Data[r.JoinField] = map[string]interface{}{
				"name" : r.JoinFieldName,
			}
		}
	} else if len(r.Parent) > 0 {
		metaData["_parent"] = r.Parent
	}
	if r.Action == ActionUpdate || !r.HardCrud {
		metaData["_retry_on_conflict"] = "4"
	}

	if r.HardCrud {
		meta[r.Action] = metaData
	} else {
		meta["update"] = metaData // all requests are update in this case
	}

	data, err := json.Marshal(meta)
	if err != nil {
		return errors.Trace(err)
	}
	if r.Action != ActionUpdate || (len(r.Data) > 1 || (len(r.JoinField) == 0 && len(r.Data) == 1)) {

		buf.Write(data)
		buf.WriteByte('\n')
	}

	switch r.Action {
	case ActionDelete:
		if !r.HardCrud {
			var del bytes.Buffer
			for k := range r.Data {
				del.WriteString("ctx._source.remove(\"")
				del.WriteString(k)
				del.WriteString("\");")
			}
			delScript := map[string]interface{}{
						"inline" : del.String(),
						"lang" : "painless",
					}
{'fraction_non_alphanumeric': 0.14267767408470927, 'fraction_numerical': 0.0034099066762383347, 'mean_word_length': 2.9665480427046265, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3051200', 'n_tokens_mistral': 2177, 'n_tokens_neox': 2002, 'n_words': 482}