<commit_before>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package costs

import (
	"context"
	"fmt"
	"net/http"
	"time"

	"github.com/trackit/jsonlog"
	"github.com/trackit/trackit2/db"
	"github.com/trackit/trackit2/es"
	"github.com/trackit/trackit2/routes"
	"github.com/trackit/trackit2/users"
	"gopkg.in/olivere/elastic.v5"
)

// esQueryParams will store the parsed query params
type esQueryParams struct {
	dateBegin   time.Time
	dateEnd     time.Time
	accountList []uint
}

// esFilter represents an elasticsearch filter
type esFilter struct {
	Key   string
	Value string
}

type esFilters = []esFilter

// queryDataTypeToEsFilters represents the different types of data
// that can be requested from ES and their associated slice of filters
var queryDataTypeToEsFilters = map[string]esFilters{
	"storage": esFilters{
		esFilter{"usageType", "*TimedStorage*"},
	},
	"requests": esFilters{
		esFilter{"usageType", "*Requests*"},
	},
	"bandwidthIn": esFilters{
		esFilter{"usageType", "*In*"},
		esFilter{"serviceCode", "AWSDataTransfer"},
	},
	"bandwidthOut": esFilters{
		esFilter{"usageType", "*Out*"},
		esFilter{"serviceCode", "AWSDataTransfer"},
	},
}

func init() {
	routes.MethodMuxer{
		http.MethodGet: routes.H(getS3CostData).With(
			db.RequestTransaction{Db: db.Db},
			users.RequireAuthenticatedUser{},
			routes.Documentation{
				Summary:     "get the s3 costs data",
				Description: "Responds with cost data based on the queryparams passed to it",
			},
			routes.QueryArgs{AwsAccountsQueryArg},
			routes.QueryArgs{DateBeginQueryArg},
			routes.QueryArgs{DateEndQueryArg},
		),
	}.H().Register("/s3_costs")
}

var (
	// AwsAccountsQueryArg allows to get the AWS Account IDs in the URL
	// Parameters with routes.QueryArgs. These AWS Account IDs will be a
	// slice of Uint stored in the routes.Arguments map with itself for key.
	AwsAccountsQueryArg = routes.QueryArg{
		Name:        "accounts",
		Type:        routes.QueryArgUintSlice{},
		Description: "The IDs for many AWS account.",
	}

	// DateBeginQueryArg allows to get the iso8601 begin date in the URL
	// Parameters with routes.QueryArgs. This date will be a
	// time.Time stored in the routes.Arguments map with itself for key.
	DateBeginQueryArg = routes.QueryArg{
		Name:        "begin",
		Type:        routes.QueryArgDate{},
		Description: "The begin date.",
	}

	// DateEndQueryArg allows to get the iso8601 begin date in the URL
	// Parameters with routes.QueryArgs. This date will be a
	// time.Time stored in the routes.Arguments map with itself for key.
	DateEndQueryArg = routes.QueryArg{
		Name:        "end",
		Type:        routes.QueryArgDate{},
		Description: "The end date.",
	}
)

// makeElasticSearchRequest prepares and run the request to retrieve usage and cost
// informations related to the queryDataType
// It will return the data, an http status code (as int) and an error.
// Because an error can be generated, but is not critical and is not needed to be known by
// the user (e.g if the index does not exists because it was not yet indexed ) the error will
// be returned, but instead of having a 500 status code, it will return the provided status code
// with empy data
func makeElasticSearchRequest(ctx context.Context, parsedParams esQueryParams,
	user users.User, queryDataType string) (*elastic.SearchResult, int, error) {
	l := jsonlog.LoggerFromContextOrDefault(ctx)
	index := es.IndexNameForUser(user, "lineitems")

	esFilters, ok := queryDataTypeToEsFilters[queryDataType]
	if ok == false {
		err := fmt.Errorf("QueryDataType '%s' not found", queryDataType)
		l.Error("Failed to retrieve s3 costs", err)
		return nil, http.StatusInternalServerError, err
	}

	searchService := GetS3UsageAndCostElasticSearchParams(
		parsedParams.accountList,
		parsedParams.dateBegin,
		parsedParams.dateEnd,
		esFilters,
		es.Client,
		index,
	)
	res, err := searchService.Do(ctx)
	if err != nil {
		if elastic.IsNotFound(err) {
			l.Warning("Query execution failed, ES index does not exists : "+index, err)
			return nil, http.StatusOK, err
		}
		l.Error("Query execution failed : "+err.Error(), nil)
		return nil, http.StatusInternalServerError, fmt.Errorf("could not execute the ElasticSearch query")
	}
	return res, http.StatusOK, nil
}

// getS3CostData returns the s3 cost data based on the query params, in JSON format.
func getS3CostData(request *http.Request, a routes.Arguments) (int, interface{}) {
	user := a[users.AuthenticatedUser].(users.User)
	parsedParams := esQueryParams{}
	parsedParams.accountList = a[AwsAccountsQueryArg].([]uint)
	parsedParams.dateBegin = a[DateBeginQueryArg].(time.Time)
	parsedParams.dateEnd = a[DateEndQueryArg].(time.Time)<commit_msg>s3/costs: make the accounts parameter optional<commit_after>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package costs

import (
	"context"
	"fmt"
	"net/http"
	"time"

	"github.com/trackit/jsonlog"
	"github.com/trackit/trackit2/db"
	"github.com/trackit/trackit2/es"
	"github.com/trackit/trackit2/routes"
	"github.com/trackit/trackit2/users"
	"gopkg.in/olivere/elastic.v5"
)

// esQueryParams will store the parsed query params
type esQueryParams struct {
	dateBegin   time.Time
	dateEnd     time.Time
	accountList []uint
}

// esFilter represents an elasticsearch filter
type esFilter struct {
	Key   string
	Value string
}

type esFilters = []esFilter

// queryDataTypeToEsFilters represents the different types of data
// that can be requested from ES and their associated slice of filters
var queryDataTypeToEsFilters = map[string]esFilters{
	"storage": esFilters{
		esFilter{"usageType", "*TimedStorage*"},
	},
	"requests": esFilters{
		esFilter{"usageType", "*Requests*"},
	},
	"bandwidthIn": esFilters{
		esFilter{"usageType", "*In*"},
		esFilter{"serviceCode", "AWSDataTransfer"},
	},
	"bandwidthOut": esFilters{
		esFilter{"usageType", "*Out*"},
		esFilter{"serviceCode", "AWSDataTransfer"},
	},
}

func init() {
	routes.MethodMuxer{
		http.MethodGet: routes.H(getS3CostData).With(
			db.RequestTransaction{Db: db.Db},
			users.RequireAuthenticatedUser{},
			routes.Documentation{
				Summary:     "get the s3 costs data",
				Description: "Responds with cost data based on the queryparams passed to it",
			},
			routes.QueryArgs{AwsAccountsQueryArg},
			routes.QueryArgs{DateBeginQueryArg},
			routes.QueryArgs{DateEndQueryArg},
		),
	}.H().Register("/s3_costs")
}

var (
	// AwsAccountsQueryArg allows to get the AWS Account IDs in the URL
	// Parameters with routes.QueryArgs. These AWS Account IDs will be a
	// slice of Uint stored in the routes.Arguments map with itself for key.
	AwsAccountsQueryArg = routes.QueryArg{
		Name:        "accounts",
		Type:        routes.QueryArgUintSlice{},
		Description: "The IDs for many AWS account.",
		Optional:    true,
	}

	// DateBeginQueryArg allows to get the iso8601 begin date in the URL
	// Parameters with routes.QueryArgs. This date will be a
	// time.Time stored in the routes.Arguments map with itself for key.
	DateBeginQueryArg = routes.QueryArg{
		Name:        "begin",
		Type:        routes.QueryArgDate{},
		Description: "The begin date.",
	}

	// DateEndQueryArg allows to get the iso8601 begin date in the URL
	// Parameters with routes.QueryArgs. This date will be a
	// time.Time stored in the routes.Arguments map with itself for key.
	DateEndQueryArg = routes.QueryArg{
		Name:        "end",
		Type:        routes.QueryArgDate{},
		Description: "The end date.",
	}
)

// makeElasticSearchRequest prepares and run the request to retrieve usage and cost
// informations related to the queryDataType
// It will return the data, an http status code (as int) and an error.
// Because an error can be generated, but is not critical and is not needed to be known by
// the user (e.g if the index does not exists because it was not yet indexed ) the error will
// be returned, but instead of having a 500 status code, it will return the provided status code
// with empy data
func makeElasticSearchRequest(ctx context.Context, parsedParams esQueryParams,
	user users.User, queryDataType string) (*elastic.SearchResult, int, error) {
	l := jsonlog.LoggerFromContextOrDefault(ctx)
	index := es.IndexNameForUser(user, "lineitems")

	esFilters, ok := queryDataTypeToEsFilters[queryDataType]
	if ok == false {
		err := fmt.Errorf("QueryDataType '%s' not found", queryDataType)
		l.Error("Failed to retrieve s3 costs", err)
		return nil, http.StatusInternalServerError, err
	}

	searchService := GetS3UsageAndCostElasticSearchParams(
		parsedParams.accountList,
		parsedParams.dateBegin,
		parsedParams.dateEnd,
		esFilters,
		es.Client,
		index,
	)
	res, err := searchService.Do(ctx)
	if err != nil {
		if elastic.IsNotFound(err) {
			l.Warning("Query execution failed, ES index does not exists : "+index, err)
			return nil, http.StatusOK, err
		}
		l.Error("Query execution failed : "+err.Error(), nil)
		return nil, http.StatusInternalServerError, fmt.Errorf("could not execute the ElasticSearch query")
	}
	return res, http.StatusOK, nil
}

// getS3CostData returns the s3 cost data based on the query params, in JSON format.
func getS3CostData(request *http.Request, a routes.Arguments) (int, interface{}) {
	user := a[users.AuthenticatedUser].(users.User)
	parsedParams := esQueryParams{
		dateBegin:   a[DateBeginQueryArg].(time.Time),
		dateEnd:     a[DateEndQueryArg].(time.Time),
		accountList: []uint{},
	}
	if a[AwsAccountsQueryArg] != nil {
		parsedParams.accountList = a[AwsAccountsQueryArg].([]uint)
	}
{'fraction_non_alphanumeric': 0.10522338717294795, 'fraction_numerical': 0.006139605176159441, 'mean_word_length': 4.0251542477456095, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2853985', 'n_tokens_mistral': 3356, 'n_tokens_neox': 3259, 'n_words': 1067}