<commit_before>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"math"
	"strconv"
	"time"
)

// Timeseries is a single time series, identified with the associated tagset.
type Timeseries struct {
	Values []float64
	TagSet TagSet
	Raw    [][]byte
}

// MarshalJSON exists to manually encode floats.
func (ts Timeseries) MarshalJSON() ([]byte, error) {
	var buffer bytes.Buffer
	var scratch [64]byte
	buffer.WriteByte('{')
	buffer.WriteString("\"tagset\":")
	tagset, err := json.Marshal(ts.TagSet)
	if err != nil {
		return []byte{}, err
	}
	buffer.Write(tagset)
	buffer.WriteByte(',')

	if ts.Raw != nil {
		buffer.WriteString("\"raw\":")
		buffer.WriteByte('[')
		first := true
		for _, raw := range ts.Raw {
			if !first {
				buffer.WriteByte(',')
			}
			buffer.WriteByte('[')
			buffer.WriteByte('"')
			base64Wrapped := base64.StdEncoding.EncodeToString(raw)
			buffer.WriteString(base64Wrapped)
			buffer.WriteByte('"')
			buffer.WriteByte(']')
			first = false
		}
		// raw, _ := json.Marshal(ts.Raw)
		buffer.WriteByte(']')
		buffer.WriteByte(',')
	}

	// buffer.WriteByte(',')
	buffer.WriteString("\"values\":")
	buffer.WriteByte('[')
	n := len(ts.Values)
	for i := 0; i < n; i++ {
		if i > 0 {
			buffer.WriteByte(',')
		}
		f := ts.Values[i]
		if math.IsInf(f, 1) {
			buffer.WriteString("null") // TODO - positive infinity
		} else if math.IsInf(f, -1) {
			buffer.WriteString("null") // TODO - negative infinity
		} else if math.IsNaN(f) {
			buffer.WriteString("null")
		} else {
			b := strconv.AppendFloat(scratch[:0], f, 'g', -1, 64)
			buffer.Write(b)
		}
	}
	buffer.WriteByte(']')
	buffer.WriteByte('}')
	return buffer.Bytes(), err
}

func (ts Timeseries) Truncate(original Timerange, timerange Timerange) (Timeseries, error) {
	return Timeseries{}, fmt.Errorf("Implement Truncate()")
}

func (ts Timeseries) Upsample(original Timerange, resolution time.Duration, interpolator func(start, end, position float64) float64) (Timeseries, error) {
	/*
		newValue := make([]float64, timerange.Slots())
		return Timeseries{
			Values: newValue,
			TagSet: ts.TagSet,
		}
	*/
	return Timeseries{}, nil
}

func (ts Timeseries) Downsample(original Timerange, newRange Timerange, bucketSampler func([]float64) float64) (Timeseries, error) {
	newValue := make([]float64, newRange.Slots())
	width := float64(newRange.Resolution()) / float64(original.Resolution())
	for i := 0; i < newRange.Slots(); i++ {
		start := int(float64(i) * width)
		end := min(int(float64(i+1)*width), original.Slots())
		fmt.Printf("start=%d,end=%d\n", start, end)
		slice := ts.Values[start:end]
		newValue[i] = bucketSampler(slice)
	}
	return Timeseries{
		Values: newValue,
		TagSet: ts.TagSet,
	}, nil
}

func min(x, y int) int {
	if x < y {
		return x
	}
	return y
}<commit_msg>delete unused methods on Timeseries<commit_after>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"math"
	"strconv"
)

// Timeseries is a single time series, identified with the associated tagset.
type Timeseries struct {
	Values []float64
	TagSet TagSet
	Raw    [][]byte
}

// MarshalJSON exists to manually encode floats.
func (ts Timeseries) MarshalJSON() ([]byte, error) {
	var buffer bytes.Buffer
	var scratch [64]byte
	buffer.WriteByte('{')
	buffer.WriteString("\"tagset\":")
	tagset, err := json.Marshal(ts.TagSet)
	if err != nil {
		return []byte{}, err
	}
	buffer.Write(tagset)
	buffer.WriteByte(',')

	if ts.Raw != nil {
		buffer.WriteString("\"raw\":")
		buffer.WriteByte('[')
		first := true
		for _, raw := range ts.Raw {
			if !first {
				buffer.WriteByte(',')
			}
			buffer.WriteByte('[')
			buffer.WriteByte('"')
			base64Wrapped := base64.StdEncoding.EncodeToString(raw)
			buffer.WriteString(base64Wrapped)
			buffer.WriteByte('"')
			buffer.WriteByte(']')
			first = false
		}
		// raw, _ := json.Marshal(ts.Raw)
		buffer.WriteByte(']')
		buffer.WriteByte(',')
	}

	// buffer.WriteByte(',')
	buffer.WriteString("\"values\":")
	buffer.WriteByte('[')
	n := len(ts.Values)
	for i := 0; i < n; i++ {
		if i > 0 {
			buffer.WriteByte(',')
		}
		f := ts.Values[i]
		if math.IsInf(f, 1) {
			buffer.WriteString("null") // TODO - positive infinity
		} else if math.IsInf(f, -1) {
			buffer.WriteString("null") // TODO - negative infinity
		} else if math.IsNaN(f) {
			buffer.WriteString("null")
		} else {
			b := strconv.AppendFloat(scratch[:0], f, 'g', -1, 64)
			buffer.Write(b)
		}
	}
	buffer.WriteByte(']')
	buffer.WriteByte('}')
	return buffer.Bytes(), err
}
{'fraction_non_alphanumeric': 0.15744831242268953, 'fraction_numerical': 0.013783353949461036, 'mean_word_length': 3.708818635607321, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6913186', 'n_tokens_mistral': 2080, 'n_tokens_neox': 1906, 'n_words': 538}