<commit_before>package integration

import (
	"engine"
	"fmt"
	"reflect"
	"strings"
	"time"

	. "integration/helpers"

	"net/http"

	influxdb "github.com/influxdb/influxdb-go"
	. "launchpad.net/gocheck"
)

type DataTestSuite struct {
	server *Server
}

var _ = Suite(&DataTestSuite{})

func (self *DataTestSuite) SetUpSuite(c *C) {
	self.server = NewServer("src/integration/test_config_single.toml", c)
}

func (self *DataTestSuite) TearDownSuite(c *C) {
	if self.server == nil {
		return
	}
	self.server.Stop()
}

func (self *DataTestSuite) TestAll(c *C) {
	t := reflect.TypeOf(self)
	v := reflect.ValueOf(self)

	names := []string{}
	setup := []Fun{}
	test := []Fun{}

	client := &DataTestClient{}

	for i := 0; i < t.NumMethod(); i++ {
		method := t.Method(i)
		// the first argument is the DataTestSuite pointer
		if method.Type.NumIn() != 2 || method.Type.NumOut() != 2 {
			continue
		}
		if method.Type.In(1) != reflect.TypeOf(c) {
			c.Logf("Skipping1 %s", method.Name)
			continue
		}

		var fun Fun
		if method.Type.Out(0) != reflect.TypeOf(fun) || method.Type.Out(1) != reflect.TypeOf(fun) {
			c.Logf("Skipping2 %s", method.Name)
			continue
		}
		m := v.MethodByName(method.Name)
		returnValues := m.Call([]reflect.Value{reflect.ValueOf(c)})
		setup = append(setup, returnValues[0].Interface().(Fun))
		test = append(test, returnValues[1].Interface().(Fun))
		names = append(names, method.Name)
	}

	c.Logf("Running %d data tests", len(names))

	for idx, _ := range setup {
		c.Logf("Initializing database for %s", names[idx])
		client.CreateDatabase(fmt.Sprintf("db%d", idx), c)
	}

	self.server.WaitForServerToSync()

	for idx, s := range setup {
		client.SetDB(fmt.Sprintf("db%d", idx))
		c.Logf("Writing data for %s", names[idx])
		s(client)
	}

	self.server.WaitForServerToSync()

	// make sure the tests don't use an idle connection, otherwise the
	// server will close it
	http.DefaultTransport.(*http.Transport).CloseIdleConnections()

	for idx, t := range test {
		client.SetDB(fmt.Sprintf("db%d", idx))
		c.Logf("Started %s", names[idx])
		t(client)
		c.Logf("Finished %s", names[idx])
	}
}

type Fun func(client Client)

// issue #518
func (self *DataTestSuite) InfiniteValues(c *C) (Fun, Fun) {
	// make sure we exceed the pointBatchSize, so we force a yield to
	// the filtering engine
	return func(client Client) {
			data := `
[
  {
    "points": [
        [1399590718, 0.0],
        [1399590718, 0.0]
    ],
    "name": "test_infinite_values",
    "columns": ["time", "value"]
  }
]`
			client.WriteJsonData(data, c, influxdb.Second)
		}, func(client Client) {
			serieses := client.RunQuery("select derivative(value) from test_infinite_values", c, "m")
			c.Assert(serieses, HasLen, 1)
			maps := ToMap(serieses[0])
			c.Assert(maps, HasLen, 1)
			c.Assert(maps[0]["derivative"], IsNil)
		}
}

// issue #512
func (self *DataTestSuite) GroupByNullValues(c *C) (Fun, Fun) {
	// make sure we exceed the pointBatchSize, so we force a yield to
	// the filtering engine
	return func(client Client) {
			data := `
[
  {
    "points": [
        ["one", null],
        ["one", 1],
        ["one", null]
    ],
    "name": "test_null_groups",
    "columns": ["column0", "column1"]
  }
]`
			client.WriteJsonData(data, c, influxdb.Second)
		}, func(client Client) {
			serieses := client.RunQuery("select count(column0) from test_null_groups group by column1", c, "m")
			c.Assert(serieses, HasLen, 1)
			maps := ToMap(serieses[0])
			c.Assert(maps, HasLen, 2)
			c.Assert(maps[0]["count"], Equals, 1.0)
			// this is an implementation detail, but nulls come last in the
			// trie
			c.Assert(maps[1]["count"], Equals, 2.0)
		}
}

// issue #389
func (self *DataTestSuite) FilteringShouldNotStopIfAllPointsDontMatch(c *C) (Fun, Fun) {
	// make sure we exceed the pointBatchSize, so we force a yield to
	// the filtering engine
	return func(client Client) {
			numberOfPoints := 1000000
			serieses := CreatePointsFromFunc("test_filtering_shouldnt_stop", 1, numberOfPoints, func(i int) float64 { return float64(numberOfPoints - i) })
			series := serieses[0]
			series.Columns = append(series.Columns, "time")
			points := series.Points
			series.Points = nil
			now := time.Now()
			for idx, point := range points {
				point = append(point, float64(now.Add(time.Duration(-idx)*time.Second).Unix()))
				series.Points = append(series.Points, point)
			}
			client.WriteData([]*influxdb.Series{series}, c, influxdb.Second)
		}, func(client Client) {
			serieses := client.RunQuery("select column0 from test_filtering_shouldnt_stop where column0 < 10", c, "m")
			fmt.Printf("serieses: %#v\n", serieses)
			c.Assert(serieses, HasLen, 1)
		}
}

// issue #413
func (self *DataTestSuite) SmallGroupByIntervals(c *C) (Fun, Fun) {
	return func(client Client) {
			serieses := CreatePoints("test_small_group_by", 1, 1)
			client.WriteData(serieses, c)
		}, func(client Client) {
			serieses := client.RunQuery("select count(column0) from test_small_group_by group by time(10)", c, "m")
			c.Assert(serieses, HasLen, 1)
			c.Assert(serieses[0].Points, HasLen, 1)
			c.Assert(serieses[0].Points[0], HasLen, 2)
			c.Assert(serieses[0].Points[0][1], Equals, 1.0)
		}
}

// issue #392
func (self *DataTestSuite) DifferentColumnsAcrossShards(c *C) (Fun, Fun) {
	return func(client Client) {
			i := 0.0
			serieses := CreatePointsFromFunc("test_different_columns_across_shards", 1, 1, func(_ int) float64 { i++; return i })
			now := time.Now().Truncate(24 * time.Hour)
			serieses[0].Columns = []string{"column0", "time"}
			serieses[0].Points[0] = append(serieses[0].Points[0], now.Unix())
			client.WriteData(serieses, c, "s")
			serieses = CreatePointsFromFunc("test_different_columns_across_shards", 2, 1, func(_ int) float64 { i++; return i })
			serieses[0].Columns = []string{"column0", "column1", "time"}
			serieses[0].Points[0] = append(serieses[0].Points[0], now.Add(-8*24*time.Hour).Unix())
			client.WriteData(serieses, c, "s")
		}, func(client Client) {
			serieses := client.RunQuery("select * from test_different_columns_across_shards", c, "s")
			c.Assert(serieses, HasLen, 1)

			maps := ToMap(serieses[0])
			c.Assert(maps[0]["column0"], Equals, 1.0)
			c.Assert(maps[0]["column1"], IsNil)
			c.Assert(maps[1]["column0"], Equals, 2.0)
			c.Assert(maps[1]["column1"], Equals, 3.0)
		}
}

// issue #392
func (self *DataTestSuite) DifferentColumnsAcrossShards2(c *C) (Fun, Fun) {
	return func(client Client) {<commit_msg>Fix #540. Add a test to make sure regex matching with literal dot work<commit_after>package integration

import (
	"engine"
	"fmt"
	"reflect"
	"strings"
	"time"

	. "integration/helpers"

	"net/http"

	influxdb "github.com/influxdb/influxdb-go"
	. "launchpad.net/gocheck"
)

type DataTestSuite struct {
	server *Server
}

var _ = Suite(&DataTestSuite{})

func (self *DataTestSuite) SetUpSuite(c *C) {
	self.server = NewServer("src/integration/test_config_single.toml", c)
}

func (self *DataTestSuite) TearDownSuite(c *C) {
	if self.server == nil {
		return
	}
	self.server.Stop()
}

func (self *DataTestSuite) TestAll(c *C) {
	t := reflect.TypeOf(self)
	v := reflect.ValueOf(self)

	names := []string{}
	setup := []Fun{}
	test := []Fun{}

	client := &DataTestClient{}

	for i := 0; i < t.NumMethod(); i++ {
		method := t.Method(i)
		// the first argument is the DataTestSuite pointer
		if method.Type.NumIn() != 2 || method.Type.NumOut() != 2 {
			continue
		}
		if method.Type.In(1) != reflect.TypeOf(c) {
			c.Logf("Skipping1 %s", method.Name)
			continue
		}

		var fun Fun
		if method.Type.Out(0) != reflect.TypeOf(fun) || method.Type.Out(1) != reflect.TypeOf(fun) {
			c.Logf("Skipping2 %s", method.Name)
			continue
		}
		m := v.MethodByName(method.Name)
		returnValues := m.Call([]reflect.Value{reflect.ValueOf(c)})
		setup = append(setup, returnValues[0].Interface().(Fun))
		test = append(test, returnValues[1].Interface().(Fun))
		names = append(names, method.Name)
	}

	c.Logf("Running %d data tests", len(names))

	for idx, _ := range setup {
		c.Logf("Initializing database for %s", names[idx])
		client.CreateDatabase(fmt.Sprintf("db%d", idx), c)
	}

	self.server.WaitForServerToSync()

	for idx, s := range setup {
		client.SetDB(fmt.Sprintf("db%d", idx))
		c.Logf("Writing data for %s", names[idx])
		s(client)
	}

	self.server.WaitForServerToSync()

	// make sure the tests don't use an idle connection, otherwise the
	// server will close it
	http.DefaultTransport.(*http.Transport).CloseIdleConnections()

	for idx, t := range test {
		client.SetDB(fmt.Sprintf("db%d", idx))
		c.Logf("Started %s", names[idx])
		t(client)
		c.Logf("Finished %s", names[idx])
	}
}

type Fun func(client Client)

// issue #518
func (self *DataTestSuite) InfiniteValues(c *C) (Fun, Fun) {
	// make sure we exceed the pointBatchSize, so we force a yield to
	// the filtering engine
	return func(client Client) {
			data := `
[
  {
    "points": [
        [1399590718, 0.0],
        [1399590718, 0.0]
    ],
    "name": "test_infinite_values",
    "columns": ["time", "value"]
  }
]`
			client.WriteJsonData(data, c, influxdb.Second)
		}, func(client Client) {
			serieses := client.RunQuery("select derivative(value) from test_infinite_values", c, "m")
			c.Assert(serieses, HasLen, 1)
			maps := ToMap(serieses[0])
			c.Assert(maps, HasLen, 1)
			c.Assert(maps[0]["derivative"], IsNil)
		}
}

// issue #512
func (self *DataTestSuite) GroupByNullValues(c *C) (Fun, Fun) {
	// make sure we exceed the pointBatchSize, so we force a yield to
	// the filtering engine
	return func(client Client) {
			data := `
[
  {
    "points": [
        ["one", null],
        ["one", 1],
        ["one", null]
    ],
    "name": "test_null_groups",
    "columns": ["column0", "column1"]
  }
]`
			client.WriteJsonData(data, c, influxdb.Second)
		}, func(client Client) {
			serieses := client.RunQuery("select count(column0) from test_null_groups group by column1", c, "m")
			c.Assert(serieses, HasLen, 1)
			maps := ToMap(serieses[0])
			c.Assert(maps, HasLen, 2)
			c.Assert(maps[0]["count"], Equals, 1.0)
			// this is an implementation detail, but nulls come last in the
			// trie
			c.Assert(maps[1]["count"], Equals, 2.0)
		}
}

// issue #389
func (self *DataTestSuite) FilteringShouldNotStopIfAllPointsDontMatch(c *C) (Fun, Fun) {
	// make sure we exceed the pointBatchSize, so we force a yield to
	// the filtering engine
	return func(client Client) {
			numberOfPoints := 1000000
			serieses := CreatePointsFromFunc("test_filtering_shouldnt_stop", 1, numberOfPoints, func(i int) float64 { return float64(numberOfPoints - i) })
			series := serieses[0]
			series.Columns = append(series.Columns, "time")
			points := series.Points
			series.Points = nil
			now := time.Now()
			for idx, point := range points {
				point = append(point, float64(now.Add(time.Duration(-idx)*time.Second).Unix()))
				series.Points = append(series.Points, point)
			}
			client.WriteData([]*influxdb.Series{series}, c, influxdb.Second)
		}, func(client Client) {
			serieses := client.RunQuery("select column0 from test_filtering_shouldnt_stop where column0 < 10", c, "m")
			fmt.Printf("serieses: %#v\n", serieses)
			c.Assert(serieses, HasLen, 1)
		}
}

// issue #413
func (self *DataTestSuite) SmallGroupByIntervals(c *C) (Fun, Fun) {
	return func(client Client) {
			serieses := CreatePoints("test_small_group_by", 1, 1)
			client.WriteData(serieses, c)
		}, func(client Client) {
			serieses := client.RunQuery("select count(column0) from test_small_group_by group by time(10)", c, "m")
			c.Assert(serieses, HasLen, 1)
			c.Assert(serieses[0].Points, HasLen, 1)
			c.Assert(serieses[0].Points[0], HasLen, 2)
			c.Assert(serieses[0].Points[0][1], Equals, 1.0)
		}
}

// issue #540
func (self *DataTestSuite) RegexMatching(c *C) (Fun, Fun) {
	return func(client Client) {
			serieses := CreatePoints("cpu.1", 1, 1)
			client.WriteData(serieses, c)
			serieses = CreatePoints("cpu-2", 1, 1)
			client.WriteData(serieses, c)
		}, func(client Client) {
			serieses := client.RunQuery("select * from /cpu\\..*/ limit 1", c, "m")
			c.Assert(serieses, HasLen, 1)
			c.Assert(serieses[0].Name, Equals, "cpu.1")
		}
}

// issue #392
func (self *DataTestSuite) DifferentColumnsAcrossShards(c *C) (Fun, Fun) {
	return func(client Client) {
			i := 0.0
			serieses := CreatePointsFromFunc("test_different_columns_across_shards", 1, 1, func(_ int) float64 { i++; return i })
			now := time.Now().Truncate(24 * time.Hour)
			serieses[0].Columns = []string{"column0", "time"}
			serieses[0].Points[0] = append(serieses[0].Points[0], now.Unix())
			client.WriteData(serieses, c, "s")
			serieses = CreatePointsFromFunc("test_different_columns_across_shards", 2, 1, func(_ int) float64 { i++; return i })
			serieses[0].Columns = []string{"column0", "column1", "time"}
			serieses[0].Points[0] = append(serieses[0].Points[0], now.Add(-8*24*time.Hour).Unix())
			client.WriteData(serieses, c, "s")
		}, func(client Client) {
			serieses := client.RunQuery("select * from test_different_columns_across_shards", c, "s")
			c.Assert(serieses, HasLen, 1)

			maps := ToMap(serieses[0])
			c.Assert(maps[0]["column0"], Equals, 1.0)
			c.Assert(maps[0]["column1"], IsNil)
			c.Assert(maps[1]["column0"], Equals, 2.0)
			c.Assert(maps[1]["column1"], Equals, 3.0)
		}
}

// issue #392
func (self *DataTestSuite) DifferentColumnsAcrossShards2(c *C) (Fun, Fun) {
	return func(client Client) {
{'fraction_non_alphanumeric': 0.1675506087995817, 'fraction_numerical': 0.023754388585941585, 'mean_word_length': 4.044461190655614, 'pattern_counts': {'":': 12, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7632267', 'n_tokens_mistral': 5412, 'n_tokens_neox': 4875, 'n_words': 1115}