<commit_before>/*
 * Copyright (C) 2015 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package elasticseach

import (
	"encoding/json"
	"errors"
	"strconv"
	"strings"

	elastigo "github.com/mattbaird/elastigo/lib"

	"github.com/redhat-cip/skydive/config"
	"github.com/redhat-cip/skydive/flow"
	"github.com/redhat-cip/skydive/logging"
	"github.com/redhat-cip/skydive/storage"
)

type ElasticSearchStorage struct {
	connection *elastigo.Conn
	indexer    *elastigo.BulkIndexer
}

func (c *ElasticSearchStorage) StoreFlows(flows []*flow.Flow) error {
	for _, flow := range flows {
		err := c.indexer.Index("skydive", "flow", flow.ID(), "", "", nil, flow)
		if err != nil {
			logging.GetLogger().Errorf("Error while indexing: %s", err.Error())
			continue
		}
	}

	return nil
}

func (c *ElasticSearchStorage) SearchFlows(filters storage.Filters) ([]*flow.Flow, error) {
	query := map[string]interface{}{
		"sort": map[string]interface{}{
			"Statistics.Last": map[string]string{
				"order": "desc",
			},
		},
		"from": 0,
		"size": 5,
	}
	if len(filters) > 0 {
		query = map[string]interface{}{
			"query": map[string]interface{}{
				"term": filters,
			},
			"sort": map[string]interface{}{
				"Statistics.Last": map[string]string{
					"order": "desc",
				},
			},
			"from": 0,
			"size": 5,
		}
	}

	q, err := json.Marshal(query)
	if err != nil {
		return nil, err
	}

	out, err := c.connection.Search("skydive", "flow", nil, string(q))
	if err != nil {
		return nil, err
	}

	flows := []*flow.Flow{}

	if out.Hits.Len() > 0 {
		for _, d := range out.Hits.Hits {
			f := new(flow.Flow)
			err := json.Unmarshal([]byte(*d.Source), f)
			if err != nil {
				return nil, err
			}

			flows = append(flows, f)
		}
	}

	return flows, nil
}

func (c *ElasticSearchStorage) initialize() error {
	c.indexer = c.connection.NewBulkIndexerErrors(10, 60)
	c.indexer.Start()

	req, err := c.connection.NewRequest("GET", "/skydive", "")
	if err != nil {
		return err
	}

	var response map[string]interface{}
	code, _, _ := req.Do(&response)
	if code == 200 {
		return nil
	}

	// template to remove the analyzer
	req, err = c.connection.NewRequest("PUT", "/skydive", "")
	if err != nil {
		return err
	}

	body := `{"mappings":{"flow":{"dynamic_templates":[`
	body += `{"notanalyzed_graph":{"match":"*Graph*","mapping":{"type":"string","index":"not_analyzed"}}},`
	body += `{"notanalyzed_graph":{"match":"*Layer*","mapping":{"type":"string","index":"not_analyzed"}}}`
	body += `]}}}`

	req.SetBodyString(body)

	code, _, err = req.Do(&response)
	if err != nil {
		return err
	}

	if code != 200 {
		return errors.New("Unable to create the skydive index: " + strconv.FormatInt(int64(code), 10))
	}

	return nil
}

var ErrBadConfig = errors.New("elasticseach : Config file is misconfigured, check elasticsearch key format")

func (c *ElasticSearchStorage) Close() {
	c.indexer.Stop()
	c.connection.Close()
}

func New() (*ElasticSearchStorage, error) {<commit_msg>ES, add a version + an alias to manage mapping changes<commit_after>/*
 * Copyright (C) 2015 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package elasticseach

import (
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"strings"

	elastigo "github.com/mattbaird/elastigo/lib"

	"github.com/redhat-cip/skydive/config"
	"github.com/redhat-cip/skydive/flow"
	"github.com/redhat-cip/skydive/logging"
	"github.com/redhat-cip/skydive/storage"
)

const indexVersion = 1

const mapping = `
{"mappings":{"flow":{"dynamic_templates":[
	{"notanalyzed_graph":{"match":"*GraphPath","mapping":{"type":"string","index":"not_analyzed"}}},
	{"notanalyzed_layers":{"match":"LayersPath","mapping":{"type":"string","index":"not_analyzed"}}},
	{"start_epoch":{"match":"Start","mapping":{"type":"date", "format": "epoch_second"}}},
	{"last_epoch":{"match":"Last","mapping":{"type":"date", "format": "epoch_second"}}}
]}}}
`

type ElasticSearchStorage struct {
	connection *elastigo.Conn
	indexer    *elastigo.BulkIndexer
}

func (c *ElasticSearchStorage) StoreFlows(flows []*flow.Flow) error {
	for _, flow := range flows {
		err := c.indexer.Index("skydive", "flow", flow.ID(), "", "", nil, flow)
		if err != nil {
			logging.GetLogger().Errorf("Error while indexing: %s", err.Error())
			continue
		}
	}

	return nil
}

func (c *ElasticSearchStorage) SearchFlows(filters storage.Filters) ([]*flow.Flow, error) {
	query := map[string]interface{}{
		"sort": map[string]interface{}{
			"Statistics.Last": map[string]string{
				"order": "desc",
			},
		},
		"from": 0,
		"size": 5,
	}
	if len(filters) > 0 {
		query = map[string]interface{}{
			"query": map[string]interface{}{
				"term": filters,
			},
			"sort": map[string]interface{}{
				"Statistics.Last": map[string]string{
					"order": "desc",
				},
			},
			"from": 0,
			"size": 5,
		}
	}

	q, err := json.Marshal(query)
	if err != nil {
		return nil, err
	}

	out, err := c.connection.Search("skydive", "flow", nil, string(q))
	if err != nil {
		return nil, err
	}

	flows := []*flow.Flow{}

	if out.Hits.Len() > 0 {
		for _, d := range out.Hits.Hits {
			f := new(flow.Flow)
			err := json.Unmarshal([]byte(*d.Source), f)
			if err != nil {
				return nil, err
			}

			flows = append(flows, f)
		}
	}

	return flows, nil
}

func (c *ElasticSearchStorage) request(method string, path string, query string, body string) (int, []byte, error) {
	req, err := c.connection.NewRequest(method, path, query)
	if err != nil {
		return 503, nil, err
	}

	if body != "" {
		req.SetBodyString(body)
	}

	var response map[string]interface{}
	return req.Do(&response)
}

func (c *ElasticSearchStorage) initialize() error {
	c.indexer = c.connection.NewBulkIndexerErrors(10, 60)
	c.indexer.Start()

	indexPath := fmt.Sprintf("/skydive_v%d", indexVersion)

	code, _, _ := c.request("GET", indexPath, "", "")
	if code == 200 {
		return nil
	}

	code, _, _ = c.request("PUT", indexPath, "", mapping)
	if code != 200 {
		return errors.New("Unable to create the skydive index: " + strconv.FormatInt(int64(code), 10))
	}

	aliases := `{"actions": [`

	code, data, _ := c.request("GET", "/_aliases", "", "")
	if code == 200 {
		var current map[string]interface{}

		err := json.Unmarshal(data, &current)
		if err != nil {
			return errors.New("Unable to parse aliases: " + err.Error())
		}

		for k := range current {
			if strings.HasPrefix(k, "skydive_") {
				remove := `{"remove":{"alias": "skydive", "index": "%s"}},`
				aliases += fmt.Sprintf(remove, k)
			}
		}
	}

	add := `{"add":{"alias": "skydive", "index": "skydive_v%d"}}]}`
	aliases += fmt.Sprintf(add, indexVersion)

	code, _, _ = c.request("POST", "/_aliases", "", aliases)
	if code != 200 {
		return errors.New("Unable to create an alias to the skydive index: " + strconv.FormatInt(int64(code), 10))
	}

	return nil
}

var ErrBadConfig = errors.New("elasticseach : Config file is misconfigured, check elasticsearch key format")

func (c *ElasticSearchStorage) Close() {
	c.indexer.Stop()
	c.connection.Close()
}

func New() (*ElasticSearchStorage, error) {
{'fraction_non_alphanumeric': 0.16604303086997194, 'fraction_numerical': 0.008185219831618335, 'mean_word_length': 4.048996458087367, 'pattern_counts': {'":': 67, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6187132', 'n_tokens_mistral': 3165, 'n_tokens_neox': 2882, 'n_words': 778}