<commit_before>package main

import (
    "testing" 
    "strconv"
    "log"
    "io/ioutil"
) 

var ds Dataset
var restService *RestService

func BenchmarkInit(b *testing.B) { 
    disableLogging()
    path := "/Users/bjorn/stallo/src/src/nowac/datastore/data"
    // Set up rest service to be used later. reset timer afterwards
    restService = Init(path)
    b.ResetTimer()

    for n := 0; n < b.N; n++ {
        ds = NewDataset(path) 
    } 
} 


var result []float64
func BenchmarkGeneExpression(b *testing.B) {
    for n := 0; n < b.N; n++ {
        result = restService.GeneExpression("hsa:23533")
    }
}


var res float64
func BenchmarkAverageDiff(b *testing.B) {
    for n := 0; n < b.N; n++ {
        res = restService.AvgDiff("hsa:23533")
    }
}

func BenchmarkStd(b *testing.B) {
    for n := 0; n < b.N; n++ {
        res = restService.Std("hsa:23533")
    }
}

func BenchmarkVariance(b *testing.B) {
    for n := 0; n < b.N; n++ {
        res = restService.Variance("hsa:23533")
    }
}

func BenchmarkSetScale(b *testing.B){ 
    for n := 0; n < b.N; n++ {
        restService.SetScale("log")
    } 
} 

var exprs, bg string
func BenchmarkBackground(b *testing.B) {
    // get expression value from prev test 
    exprs = strconv.FormatFloat(result[0],'f',-1, 64)

    for n := 0; n < b.N; n++ {
        bg = restService.Bg("hsa:23533", exprs)
    }
}

func disableLogging() {
    log.SetOutput(ioutil.Discard) 
} 

<commit_msg>benchmarking datastore.<commit_after>package main

import (
    "testing" 
    "strconv"
    "log"
    "io/ioutil"
    "strings"
    "fmt"
    "time" 
    "net/http"
    "code.google.com/p/gorest"
    
) 

var ds Dataset
var restService *RestService

func BenchmarkInit(b *testing.B) { 
    disableLogging()
    path := "/Users/bjorn/stallo/src/src/nowac/datastore/data"
    // Set up rest service to be used later. reset timer afterwards
    restService = Init(path)
    b.ResetTimer()

    for n := 0; n < b.N; n++ {
        ds = NewDataset(path) 
    } 
} 


var result []float64
func BenchmarkGeneExpression(b *testing.B) {
    for n := 0; n < b.N; n++ {
        result = restService.GeneExpression("hsa:23533")
    }
}


var res float64
func BenchmarkAverageDiff(b *testing.B) {
    for n := 0; n < b.N; n++ {
        res = restService.AvgDiff("hsa:23533")
    }
}

func BenchmarkStd(b *testing.B) {
    for n := 0; n < b.N; n++ {
        res = restService.Std("hsa:23533")
    }
}

func BenchmarkVariance(b *testing.B) {
    for n := 0; n < b.N; n++ {
        res = restService.Variance("hsa:23533")
    }
}

func BenchmarkSetScale(b *testing.B){ 
    for n := 0; n < b.N; n++ {
        restService.SetScale("log")
    } 
} 

var exprs, bg string
func BenchmarkBackground(b *testing.B) {
    // get expression value from prev test 
    exprs = strconv.FormatFloat(result[0],'f',-1, 64)

    for n := 0; n < b.N; n++ {
        bg = restService.Bg("hsa:23533", exprs)
    }
}

func disableLogging() {
    log.SetOutput(ioutil.Discard) 
} 


var genes = [...]string{"hsa:1630", "hsa:836", "hsa:842", "hsa:836", "hsa:999", "hsa:1499", "hsa:51384 hsa:54361 hsa:7471 hsa:7472 hsa:7473 hsa2 hsa:7483 hsa:7484 hsa:80326 hsa:81029 hsa:89780", "hsa:1495 hsa:1496 hsa:29119", "hsa:1499", "hsa:3688", "hsa:1855 hsa:1856 hsa:1857", "hsa:10319 hsa:1282 hsa:1284 hsa:1285 hsa:1286 hsa:3911 hsa:3912 hsa:3913 hsa:3914 hsa:3915 hsa:3918", "hsa:3655 hsa:3673 hsa:3674 hsa:3675 hsa:3685", "hsa:5747", "hsa:5728", "hsa:11211 hsa:2535 hsa:7855 hsa:7976 hsa:8321 hsa:8322 hsa:8323 hsa:8324 hsa:8325 hsa:8326", "hsa:26060", "hsa:25 hsa:613", "hsa:1398 hsa:1399", "hsa:23624 hsa:867 hsa:868", "hsa:1398 hsa:1399", "hsa:23533 hsa:5290 hsa:5291 hsa:5293 hsa:5294 hsa:5295 hsa:5296 hsa:8503", "hsa:6776 hsa:6777", "hsa:3716", "hsa:1956", "hsa:2064", "hsa:5156 hsa:5159", "hsa:3480", "hsa:3815", "hsa:2322", "hsa:4233", "hsa:2260 hsa:2261 hsa:2263", "hsa:10342 hsa:4914 hsa:7170 hsa:7175", "hsa:7039", "hsa:1950", "hsa:3082", "hsa:5154 hsa:5155", "hsa:3479", "hsa:367", "hsa:3320 hsa:3326 hsa:7184", "hsa:367", "hsa:367", "hsa:2932", "hsa:8312 hsa:8313", "hsa:10297 hsa:324", "hsa:1147 hsa:3551 hsa:8517", "hsa:2475", "hsa:842", "hsa:572", "hsa:4193", "hsa:1027", "hsa:598", "hsa:6774", "hsa:6772", "hsa:2885", "hsa:6654 hsa:6655", "hsa:3265 hsa:3845 hsa:4893", "hsa:369 hsa:5894 hsa:673", "hsa:5604 hsa:5605", "hsa:83593", "hsa:11186", "hsa:4792", "hsa:4790 hsa:4791 hsa:5970", "hsa:6789", "hsa:595", "hsa:5979 hsa:8030 hsa:8031", "hsa:11186", "hsa:5335 hsa:5336", "hsa:5900", "hsa:5898 hsa:5899", "hsa:387 hsa:5879 hsa:5880 hsa:5881", "hsa:10928", "hsa:5337", "hsa:5599 hsa:5601 hsa:5602", "hsa:5879 hsa:5880 hsa:5881 hsa:998", "hsa:3725", "hsa:2353", "hsa:4609", "hsa:7157", "hsa:51176 hsa:6932 hsa:6934 hsa:83439", "hsa:5578 hsa:5579 hsa:5582", "hsa:6469", "hsa:5727", "hsa:6608", "hsa:27148", "hsa:51684", "hsa:2271", "hsa:112398 hsa:112399 hsa:54583", "hsa:6923", "hsa:6921", "hsa:7428", "hsa:8453", "hsa:9978", "hsa:2034 hsa:3091", "hsa:405 hsa:9915", "hsa:1387 hsa:2033", "hsa:54205", "hsa:675", "hsa:5888", "hsa:4292"}

var reply string

func BenchmarkManyGenes(b *testing.B){




    disableLogging() 
    path := "/Users/bjorn/stallo/src/src/nowac/datastore/data"
    // Set up rest service to be used later. reset timer afterwards
    restService = Init(path)


    fmt.Println("HERE WE ARE AGAIN")
    gorest.RegisterService(restService)

    http.Handle("/", gorest.Handle())
    go http.ListenAndServe(":8888", nil)


    startup := time.Now()

    b.ResetTimer()

    numRuns := 3000
    for n := 0; n < b.N; n++ {
        for j := 0; j < numRuns; j++ {
            sta := time.Now() 
            for _, gene := range(genes){
                g := strings.Split(gene, " ")[0]
                reply = get("http://localhost:8888/gene/"+g)
                reply = get("http://localhost:8888/gene/"+g+"/avg")
            }
          elapsed := time.Since(sta)
          fmt.Println("Done with genes in ",elapsed.Seconds(),"seconds. (",
                        time.Since(startup)," total runtime)")

         log.Println(" --------------------",j,"-----------------")
       }
    }
}



func get(URL string) string { 
    resp, err := http.Get(URL)
    
    if err != nil { 
        fmt.Println("HTTP GET ERROR:", err)
        return "ERROR" 
    }

    defer resp.Body.Close()

    body, err := ioutil.ReadAll(resp.Body) 

    if err != nil {
        log.Print("Reading response body went bad. ", err); 
    
    } 

    return string(body)
    
} 
{'fraction_non_alphanumeric': 0.18563058896083873, 'fraction_numerical': 0.13444341658957756, 'mean_word_length': 3.0850125944584383, 'pattern_counts': {'":': 1, '<': 19, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2833675', 'n_tokens_mistral': 3288, 'n_tokens_neox': 2637, 'n_words': 564}