<commit_before>package parser

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"regexp"
	"strings"
)

type NodeStats struct {
	Total_time_micros    int64   `json:"total_time_micros"`
	Op_per_second        float64 `json:"op_per_second"`
	Op_count             int64   `json:"op_count"`
	Op_errors            int64   `json:"op_errors"`
	Op_retries           int64   `json:"op_retries"`
	Op_retry_time_micros int64   `json:"op_retry_time_micros"`
	Op_median            int64   `json:"op_median"`
	Op_lat_avg_micros    int64   `json:"op_lat_avg_micros"`
	Op_lat_min_micros    int64   `json:"op_lat_min_micros"`
	Op_lat_max_micros    int64   `json:"op_lat_max_micros"`
	Op_lat_variance      int64   `json:"op_lat_variance"`
	Op_lat_avg_95th      int64   `json:"op_lat_avg_95th"`
	Op_lat_avg_99th      int64   `json:"op_lat_avg_99th"`
	Op_lat_total_micros  int64   `json:"op_lat_total_micros"`
}

type StatsSummary struct {
	AllNodes NodeStats              `json:"all_nodes"`
	Nodes    []map[string]NodeStats `json:"nodes"`
}

func ProcessMongoSIMResult(file string) StatsSummary {
	result := StatsSummary{}

	f, err := ioutil.ReadFile(file)

	if err != nil {
		log.Fatal("cannot open file " + file)
	}

	lines := strings.Split(string(f), "\n")
	re := regexp.MustCompile("==== final metrics ====")

	for i := len(lines) - 1; i >= 0; i-- {
		if re.MatchString(lines[i]) {
			json.Unmarshal([]byte(lines[i+1]), &result)

			return result
		}
	}

	return StatsSummary{AllNodes: NodeStats{Op_per_second: 100}}
}

func ProcessSysbenchResult(file string) (string, []string, map[string]string) {
	var cum string
	var trend []string
	att := make(map[string]string)

	att["test-type"] = "sysbench"
	att["thread"] = "64"

	f, err := ioutil.ReadFile(file)

	if err != nil {
		log.Fatal("cannot open file " + file)
	}

	lines := strings.Split(string(f), "\n")

	re := regexp.MustCompile("seconds : cum tps=([0-9.]+) : int tps=([0-9.]+) : cum ips=[0-9.]+ : int ips=[0-9.]+")

	// find the cumulative number
	for i := len(lines) - 1; i >= 0; i-- {
		t := re.FindStringSubmatch(lines[i])
		if len(t) > 0 {
			cum = t[1]
			break
		} else {
			// no match, just skip
		}
	}

	// get the historical interval number
	for i := 0; i < len(lines); i++ {
		t := re.FindStringSubmatch(lines[i])
		if len(t) > 0 {<commit_msg>shall handle comma in number<commit_after>package parser

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"regexp"
	"strings"
)

type NodeStats struct {
	Total_time_micros    int64   `json:"total_time_micros"`
	Op_per_second        float64 `json:"op_per_second"`
	Op_count             int64   `json:"op_count"`
	Op_errors            int64   `json:"op_errors"`
	Op_retries           int64   `json:"op_retries"`
	Op_retry_time_micros int64   `json:"op_retry_time_micros"`
	Op_median            int64   `json:"op_median"`
	Op_lat_avg_micros    int64   `json:"op_lat_avg_micros"`
	Op_lat_min_micros    int64   `json:"op_lat_min_micros"`
	Op_lat_max_micros    int64   `json:"op_lat_max_micros"`
	Op_lat_variance      int64   `json:"op_lat_variance"`
	Op_lat_avg_95th      int64   `json:"op_lat_avg_95th"`
	Op_lat_avg_99th      int64   `json:"op_lat_avg_99th"`
	Op_lat_total_micros  int64   `json:"op_lat_total_micros"`
}

type StatsSummary struct {
	AllNodes NodeStats              `json:"all_nodes"`
	Nodes    []map[string]NodeStats `json:"nodes"`
}

func ProcessMongoSIMResult(file string) StatsSummary {
	result := StatsSummary{}

	f, err := ioutil.ReadFile(file)

	if err != nil {
		log.Fatal("cannot open file " + file)
	}

	lines := strings.Split(string(f), "\n")
	re := regexp.MustCompile("==== final metrics ====")

	for i := len(lines) - 1; i >= 0; i-- {
		if re.MatchString(lines[i]) {
			json.Unmarshal([]byte(lines[i+1]), &result)

			return result
		}
	}

	return StatsSummary{AllNodes: NodeStats{Op_per_second: 100}}
}

func ProcessSysbenchResult(file string) (string, []string, map[string]string) {
	var cum string
	var trend []string
	att := make(map[string]string)

	att["test-type"] = "sysbench"
	att["thread"] = "64"

	f, err := ioutil.ReadFile(file)

	if err != nil {
		log.Fatal("cannot open file " + file)
	}

	lines := strings.Split(string(f), "\n")

	re := regexp.MustCompile("seconds : cum tps=([0-9.,]+) : int tps=([0-9.,]+) : cum ips=[0-9.,]+ : int ips=[0-9.,]+")

	// find the cumulative number
	for i := len(lines) - 1; i >= 0; i-- {
		t := re.FindStringSubmatch(lines[i])
		if len(t) > 0 {
			cum = t[1]
			break
		} else {
			// no match, just skip
		}
	}

	// get the historical interval number
	for i := 0; i < len(lines); i++ {
		t := re.FindStringSubmatch(lines[i])
		if len(t) > 0 {
{'fraction_non_alphanumeric': 0.1533116178067318, 'fraction_numerical': 0.0251900108577633, 'mean_word_length': 3.5830845771144277, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1151885', 'n_tokens_mistral': 1936, 'n_tokens_neox': 1776, 'n_words': 370}