<commit_before>package perfdata

import (
	"github.com/anchor/dataframe"
	"strconv"
	"time"
)

// Return a struct of key/value pairs in the form of a slice of pointers
// to dataframe.DataFrame_Tags.
func getDataSource(datum RenderedPerfDatumValue, unit UOM) []*dataframe.DataFrame_Tag {
	source := make([]*dataframe.DataFrame_Tag, 0)
	source = append(source, dataframe.NewDataFrameTag("hostname", datum.Hostname))
	source = append(source, dataframe.NewDataFrameTag("service_name", datum.ServiceName))
	source = append(source, dataframe.NewDataFrameTag("metric", datum.Metric))
	source = append(source, dataframe.NewDataFrameTag("uom", unit.String()))
	return source
}

// Take a RenderedPerfDatumValue and return a DataFrame suitable for
// serialisation. This includes parsing the value of the metric (which
// is still a raw string in a RenderedPerfDatumValue) into its correct
// units.
func NewDataFrame(datum RenderedPerfDatumValue) (*dataframe.DataFrame, error) {
	frame := new(dataframe.DataFrame)
	// Need a variable to get an address
	frameTime := time.Unix(int64(datum.Timestamp), 0)
	timestamp := uint64(frameTime.UnixNano())
	rawValue, suffix := splitValueUOM(datum.Value)
	uom, err := ParseUOM(suffix)
	if err != nil {
		return nil, err
	}
	dataSource := getDataSource(datum, uom)
	switch {
	case isMicrosecondUOM(uom):
		value, err := parseMicroseconds(rawValue, uom)
		if err != nil {
			return nil, err
		}
		payload := dataframe.DataFrame_REAL
		frame.Payload = &payload
		frame.ValueMeasurement = &value
	case isByteUOM(uom):
		value, err := parseBytes(rawValue, uom)
		if err != nil {
			return nil, err
		}
		payload := dataframe.DataFrame_NUMBER
		frame.Payload = &payload
		frame.ValueNumeric = &value
	case uom == Counter:
		payload := dataframe.DataFrame_EMPTY
		frame.Payload = &payload
	// If we're given an empty string as UOM, we assume
	// it's an unadorned float and parse it accordingly. If that
	// fails, we treat it as empty/'counter'.
	case uom == NullUnit:
		value, err := strconv.ParseFloat(rawValue, 64)
		if err != nil {
			payload := dataframe.DataFrame_EMPTY
			frame.Payload = &payload
		} else {
			payload := dataframe.DataFrame_REAL
			frame.Payload = &payload
			frame.ValueMeasurement = &value
		}
	}
	frame.Source = dataSource
	frame.Timestamp = &timestamp
	return frame, nil
}<commit_msg>Fix bug in UOM parsing<commit_after>package perfdata

import (
	"github.com/anchor/dataframe"
	"strconv"
	"time"
)

// Return a struct of key/value pairs in the form of a slice of pointers
// to dataframe.DataFrame_Tags.
func getDataSource(datum RenderedPerfDatumValue, unit UOM) []*dataframe.DataFrame_Tag {
	source := make([]*dataframe.DataFrame_Tag, 0)
	source = append(source, dataframe.NewDataFrameTag("hostname", datum.Hostname))
	source = append(source, dataframe.NewDataFrameTag("service_name", datum.ServiceName))
	source = append(source, dataframe.NewDataFrameTag("metric", datum.Metric))
	source = append(source, dataframe.NewDataFrameTag("uom", unit.String()))
	return source
}

// Take a RenderedPerfDatumValue and return a DataFrame suitable for
// serialisation. This includes parsing the value of the metric (which
// is still a raw string in a RenderedPerfDatumValue) into its correct
// units.
func NewDataFrame(datum RenderedPerfDatumValue) (*dataframe.DataFrame, error) {
	frame := new(dataframe.DataFrame)
	// Need a variable to get an address
	frameTime := time.Unix(int64(datum.Timestamp), 0)
	timestamp := uint64(frameTime.UnixNano())
	rawValue, suffix := splitValueUOM(datum.Value)
	uom, err := ParseUOM(suffix)
	if err != nil {
		return nil, err
	}
	switch {
	case isMicrosecondUOM(uom):
		value, err := parseMicroseconds(rawValue, uom)
		if err != nil {
			return nil, err
		}
		payload := dataframe.DataFrame_REAL
		frame.Payload = &payload
		frame.ValueMeasurement = &value
		uom = Microseconds
	case isByteUOM(uom):
		value, err := parseBytes(rawValue, uom)
		if err != nil {
			return nil, err
		}
		payload := dataframe.DataFrame_NUMBER
		frame.Payload = &payload
		frame.ValueNumeric = &value
		uom = Bytes
	case uom == Counter:
		payload := dataframe.DataFrame_EMPTY
		frame.Payload = &payload
	// If we're given an empty string as UOM, we assume
	// it's an unadorned float and parse it accordingly. If that
	// fails, we treat it as empty/'counter'.
	case uom == NullUnit:
		value, err := strconv.ParseFloat(rawValue, 64)
		if err != nil {
			payload := dataframe.DataFrame_EMPTY
			frame.Payload = &payload
		} else {
			payload := dataframe.DataFrame_REAL
			frame.Payload = &payload
			frame.ValueMeasurement = &value
		}
	}
	dataSource := getDataSource(datum, uom)
	frame.Source = dataSource
	frame.Timestamp = &timestamp
	return frame, nil
}
{'fraction_non_alphanumeric': 0.11016949152542373, 'fraction_numerical': 0.003389830508474576, 'mean_word_length': 4.407789232531501, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3998934', 'n_tokens_mistral': 1643, 'n_tokens_neox': 1540, 'n_words': 432}