<commit_before># -*- coding: utf-8 -*-
import os
import unittest
from influxdb import InfluxDBClient
import influxdb.exceptions
import graphite_influxdb
import graphite_influxdb.utils
from graphite_influxdb.utils import Query
from graphite_influxdb.constants import SERIES_LOADER_MUTEX_KEY, \
     MEMCACHE_SERIES_DEFAULT_TTL, LOADER_LIMIT
import datetime
import time
try:
    import memcache
except ImportError:
    pass
import sys

os.environ['TZ'] = 'UTC'

class InfluxGraphTemplatesIntegrationTestCase(unittest.TestCase):
    """Test node lookup and data retrieval when using tags on and Graphite
    templates configured on InfluxGraph"""
    
    def setUp(self):
        self.metric_prefix = "template_integration_test"
        self.tags = {
            'a_test_type': self.metric_prefix,
            'b_host': 'localhost',
            'env': 'int',
            'region': 'the_west',
            }
        self.measurements = ['cpu', 'memory', 'load', 'iops']
        self.graphite_series = ["%s.%s" % (self.metric_prefix, ".".join(
            self.tags.keys() + [m])) for m in self.measurements]
        # 
        # import ipdb; ipdb.set_trace()
        self.step, self.num_datapoints, self.db_name = 60, 2, 'integration_test'
        self.start_time, self.end_time = (datetime.datetime.utcnow() - datetime.timedelta(hours=1)), \
          datetime.datetime.utcnow()
        self.steps = int(round((int(self.end_time.strftime("%s")) - \
                                int(self.start_time.strftime("%s"))) * 1.0 / self.step)) + 1
        self.config = { 'influxdb': {
            'host' : 'localhost',
            'port' : 8086,
            'user' : 'root',
            'pass' : '<PASSWORD>',
            'db' : self.db_name,
            'log_level' : 'debug',
            'templates' : [
                "%s type.host.measurement* env=int,region=the_west" % (self.metric_prefix,)
                ],
            },
            # 'search_index': 'index',
        }
        self.client = InfluxDBClient(database=self.db_name)
        self.default_nodes_limit = LOADER_LIMIT
        self.setup_db()
        self.finder = graphite_influxdb.InfluxdbFinder(self.config)

    def setup_db(self):
        try:
            self.client.drop_database(self.db_name)
        except influxdb.exceptions.InfluxDBClientError:
            pass
        self.client.create_database(self.db_name)
        data = [{
            "measurement": measurement,
            "tags": self.tags,
            "time": _time,
            "fields": {
                "value": 1,
                }
            }
            for measurement in self.measurements
            for _time in [
                (self.end_time - datetime.timedelta(minutes=30)).strftime("%Y-%m-%dT%H:%M:%SZ"),
                (self.end_time - datetime.timedelta(minutes=2)).strftime("%Y-%m-%dT%H:%M:%SZ"),
                ]]
        self.assertTrue(self.client.write_points(data))

    def tearDown(self):
        self.client.drop_database(self.db_name)

    def test_templated_index_find(self):
        query = Query('*')
        nodes = [n.path for n in self.finder.find_nodes(query)]
        expected = [self.metric_prefix]
        self.assertEqual(nodes, expected,
                         msg="Got root branch query result %s - wanted %s" % (
                             nodes, expected,))

if __name__ == '__main__':
    unittest.main()<commit_msg>Added more node finding tests on templated data<commit_after># -*- coding: utf-8 -*-
import os
import unittest
from influxdb import InfluxDBClient
import influxdb.exceptions
import graphite_influxdb
import graphite_influxdb.utils
from graphite_influxdb.utils import Query
from graphite_influxdb.constants import SERIES_LOADER_MUTEX_KEY, \
     MEMCACHE_SERIES_DEFAULT_TTL, LOADER_LIMIT
import datetime
import time
try:
    import memcache
except ImportError:
    pass
import sys

os.environ['TZ'] = 'UTC'

class InfluxGraphTemplatesIntegrationTestCase(unittest.TestCase):
    """Test node lookup and data retrieval when using tags on and Graphite
    templates configured on InfluxGraph"""
    
    def setUp(self):
        self.metric_prefix = "template_integration_test"
        self.tags = {
            'a_test_type': self.metric_prefix,
            'b_host': 'localhost',
            'env': 'int',
            'region': 'the_west',
            }
        self.measurements = ['cpu', 'memory', 'load', 'iops']
        self.graphite_series = ["%s.%s" % (self.metric_prefix, ".".join(
            self.tags.values() + [m])) for m in self.measurements]
        # 
        # import ipdb; ipdb.set_trace()
        self.step, self.num_datapoints, self.db_name = 60, 2, 'integration_test'
        self.start_time, self.end_time = (datetime.datetime.utcnow() - datetime.timedelta(hours=1)), \
          datetime.datetime.utcnow()
        self.steps = int(round((int(self.end_time.strftime("%s")) - \
                                int(self.start_time.strftime("%s"))) * 1.0 / self.step)) + 1
        self.config = { 'influxdb': {
            'host' : 'localhost',
            'port' : 8086,
            'user' : 'root',
            'pass' : '<PASSWORD>',
            'db' : self.db_name,
            'log_level' : 'debug',
            'templates' : [
                "%s type.host.measurement* env=int,region=the_west" % (self.metric_prefix,)
                ],
            },
            # 'search_index': 'index',
        }
        self.client = InfluxDBClient(database=self.db_name)
        self.default_nodes_limit = LOADER_LIMIT
        self.setup_db()
        self.finder = graphite_influxdb.InfluxdbFinder(self.config)

    def setup_db(self):
        try:
            self.client.drop_database(self.db_name)
        except influxdb.exceptions.InfluxDBClientError:
            pass
        self.client.create_database(self.db_name)
        data = [{
            "measurement": measurement,
            "tags": self.tags,
            "time": _time,
            "fields": {
                "value": 1,
                }
            }
            for measurement in self.measurements
            for _time in [
                (self.end_time - datetime.timedelta(minutes=30)).strftime("%Y-%m-%dT%H:%M:%SZ"),
                (self.end_time - datetime.timedelta(minutes=2)).strftime("%Y-%m-%dT%H:%M:%SZ"),
                ]]
        self.assertTrue(self.client.write_points(data))

    def tearDown(self):
        self.client.drop_database(self.db_name)

    def test_templated_index_find(self):
        query = Query('*')
        nodes = [n.path for n in self.finder.find_nodes(query)]
        expected = [self.metric_prefix]
        self.assertEqual(nodes, expected,
                         msg="Got root branch query result %s - wanted %s" % (
                             nodes, expected,))
        query = Query("%s.*" % (self.metric_prefix,))
        nodes = [n.path for n in self.finder.find_nodes(query)]
        expected = [".".join(list(set([d for e in [
            m.split('.')[0:2] for m in self.graphite_series] for d in e])))]
        self.assertEqual(nodes, expected,
                         msg="Got root branch query result %s - wanted %s" % (
                             nodes, expected,))
        # TODO - figure out how to handle default tags
        query = Query("%s.%s.*" % (self.metric_prefix, self.tags['b_host']))
        nodes = [n.path for n in self.finder.find_nodes(query)]
        expected = self.measurements
        self.assertEqual(nodes, expected,
                         msg="Got root branch query result %s - wanted %s" % (
                             nodes, expected,))

    def test_templated_data_query(self):
        pass

if __name__ == '__main__':
    unittest.main()
{'fraction_non_alphanumeric': 0.12631169840652934, 'fraction_numerical': 0.00440471563674051, 'mean_word_length': 2.0671434247119587, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4143409', 'n_tokens_mistral': 2404, 'n_tokens_neox': 2312, 'n_words': 468}