<commit_before>// Copyright 2016,2017 <NAME> <<EMAIL>>
// and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package storage
package memory

import (
	"fmt"
	"net/url"
	"regexp"
	"time"

	"github.com/MohawkTSDB/mohawk/storage"
)

type TimeValuePair struct {
	timeStamp int64
	value     float64
}

type TimeSeries struct {
	tags      map[string]string
	data      []TimeValuePair
	lastValue TimeValuePair
}

type Tenant struct {
	ts map[string]*TimeSeries
}

type Backend struct {
	timeGranularitySec int64
	timeRetentionSec   int64
	timeLastSec        int64

	tenant map[string]*Tenant
}

// Backend functions
// Required by storage interface

func (r Backend) Name() string {
	return "Backend-Memory"
}

func (r *Backend) Open(options url.Values) {
	// set last entry time
	r.timeLastSec = 0
	// set time granularity to 30 sec
	r.timeGranularitySec = 30
	// set time retention to 7 days
	r.timeRetentionSec = 7 * 24 * 60 * 60

	// open db connection
	r.tenant = make(map[string]*Tenant, 0)

	// start a maintenance worker that will clean the db periodically
	go r.maintenance()
}

func (r Backend) GetTenants() []storage.Tenant {
	res := make([]storage.Tenant, 0, len(r.tenant))

	// return a list of tenants
	for key := range r.tenant {
		res = append(res, storage.Tenant{Id: key})
	}

	return res
}

func (r Backend) GetItemList(tenant string, tags map[string]string) []storage.Item {
	res := make([]storage.Item, 0)
	t, ok := r.tenant[tenant]

	// check tenant
	if !ok {
		return res
	}

	for key, ts := range t.ts {
		if hasMatchingTag(tags, ts.tags) {
			lastValue := storage.DataItem{
				Timestamp: ts.lastValue.timeStamp,
				Value:     ts.lastValue.value,
			}

			res = append(res, storage.Item{
				Id:         key,
				Type:       "gauge",
				Tags:       ts.tags,
				LastValues: []storage.DataItem{lastValue},
			})
		}
	}

	// filter using tags
	// 	if we have a list of _all_ items, we need to filter them by tags
	// 	if the list is already filtered, we do not need to re-filter it
	if len(tags) > 0 {
		for key, value := range tags {
			res = storage.FilterItems(res, func(i storage.Item) bool {
				r, _ := regexp.Compile("^" + value + "$")
				return r.MatchString(i.Tags[key])
			})
		}
	}

	return res
}

func (r *Backend) GetRawData(tenant string, id string, end int64, start int64, limit int64, order string) []storage.DataItem {
	res := make([]storage.DataItem, 0)

	arraySize := r.timeRetentionSec / r.timeGranularitySec
	pStart := r.getPosForTimestamp(start)
	pEnd := r.getPosForTimestamp(end)

	// make sure start and end times is in the retention time
	start, end = r.checkTimespan(start, end)

	// sanity check pEnd
	if pEnd <= pStart {
		pEnd += arraySize
	}

	// check if tenant and id exists, create them if necessary
	r.checkID(tenant, id)

	// fill data out array
	count := int64(0)
	for i := pEnd; count < limit && i >= pStart; i-- {
		d := r.tenant[tenant].ts[id].data[i%arraySize]
		// if this is a valid point
		if d.timeStamp < end && d.timeStamp >= start {
			count++
			res = append(res, storage.DataItem{
				Timestamp: d.timeStamp,
				Value:     d.value,
			})
		}
	}

	// order
	if order == "ASC" {
		for i := 0; i < len(res)/2; i++ {
			j := len(res) - i - 1
			res[i], res[j] = res[j], res[i]
		}
	}

	return res
}

func (r Backend) GetStatData(tenant string, id string, end int64, start int64, limit int64, order string, bucketDuration int64) []storage.StatItem {
	res := make([]storage.StatItem, 0)

	// make sure start and end times is in the retention time
	start, end = r.checkTimespan(start, end)

	// bucketDuration can't be smaller then granularity
	if bucketDuration < r.timeGranularitySec {
		bucketDuration = r.timeGranularitySec
	}<commit_msg>Update memory.go<commit_after>// Copyright 2016,2017 <NAME> <<EMAIL>>
// and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package storage
package memory

import (
	"fmt"
	"net/url"
	"regexp"
	"time"

	"github.com/MohawkTSDB/mohawk/storage"
)

type TimeValuePair struct {
	timeStamp int64
	value     float64
}

type TimeSeries struct {
	tags      map[string]string
	data      []TimeValuePair
	lastValue TimeValuePair
}

type Tenant struct {
	ts map[string]*TimeSeries
}

type Backend struct {
	timeGranularitySec int64
	timeRetentionSec   int64
	timeLastSec        int64

	tenant map[string]*Tenant
}

// Backend functions
// Required by storage interface

func (r Backend) Name() string {
	return "Backend-Memory"
}

func (r *Backend) Open(options url.Values) {
	// set last entry time
	r.timeLastSec = 0
	// set time granularity to 30 sec
	r.timeGranularitySec = 30
	// set time retention to 7 days
	r.timeRetentionSec = 7 * 24 * 60 * 60

	// open db connection
	r.tenant = make(map[string]*Tenant, 0)

	// start a maintenance worker that will clean the db periodically
	go r.maintenance()
}

func (r Backend) GetTenants() []storage.Tenant {
	res := make([]storage.Tenant, 0, len(r.tenant))

	// return a list of tenants
	for key := range r.tenant {
		res = append(res, storage.Tenant{Id: key})
	}

	return res
}

func (r Backend) GetItemList(tenant string, tags map[string]string) []storage.Item {
	res := make([]storage.Item, 0)
	t, ok := r.tenant[tenant]

	// check tenant
	if !ok {
		return res
	}

	for key, ts := range t.ts {
		if hasMatchingTag(tags, ts.tags) {
			lastValue := storage.DataItem{
				Timestamp: ts.lastValue.timeStamp,
				Value:     ts.lastValue.value,
			}

			res = append(res, storage.Item{
				Id:         key,
				Type:       "gauge",
				Tags:       ts.tags,
				LastValues: []storage.DataItem{lastValue},
			})
		}
	}

	// filter using tags
	// 	if we have a list of _all_ items, we need to filter them by tags
	// 	if the list is already filtered, we do not need to re-filter it
	if len(tags) > 0 {
		for key, value := range tags {
			res = storage.FilterItems(res, func(i storage.Item) bool {
				r, _ := regexp.Compile("^" + value + "$")
				return r.MatchString(i.Tags[key])
			})
		}
	}

	return res
}

func (r *Backend) GetRawData(tenant string, id string, end int64, start int64, limit int64, order string) []storage.DataItem {
	res := make([]storage.DataItem, 0)

	arraySize := r.timeRetentionSec / r.timeGranularitySec
	pStart := r.getPosForTimestamp(start)
	pEnd := r.getPosForTimestamp(end)

	// make sure start and end times is in the retention time
	start, end = r.checkTimespan(start, end)

	// sanity check pEnd
	if pEnd <= pStart {
		pEnd += arraySize
	}

	// check if tenant and id exists, create them if necessary
	r.checkID(tenant, id)

	// fill data out array
	count := int64(0)
	for i := pEnd; count < limit && i >= pStart; i-- {
		d := r.tenant[tenant].ts[id].data[i%arraySize]

		// if this is a valid point
		if d.timeStamp < end && d.timeStamp >= start {
			count++
			res = append(res, storage.DataItem{
				Timestamp: d.timeStamp,
				Value:     d.value,
			})
		}
	}

	// order
	if order == "ASC" {
		for i := 0; i < len(res)/2; i++ {
			j := len(res) - i - 1
			res[i], res[j] = res[j], res[i]
		}
	}

	return res
}

func (r Backend) GetStatData(tenant string, id string, end int64, start int64, limit int64, order string, bucketDuration int64) []storage.StatItem {
	res := make([]storage.StatItem, 0)

	// make sure start and end times is in the retention time
	start, end = r.checkTimespan(start, end)

	// bucketDuration can't be smaller then granularity
	if bucketDuration < r.timeGranularitySec {
		bucketDuration = r.timeGranularitySec
	}
{'fraction_non_alphanumeric': 0.12022180273714016, 'fraction_numerical': 0.014393581878244455, 'mean_word_length': 3.184106614017769, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6311114', 'n_tokens_mistral': 3117, 'n_tokens_neox': 2866, 'n_words': 945}