<commit_before>#!/usr/bin/env python
# -*- coding: utf8 -*-

"""Check my Website to InfluxDB

Usage:
  cmws2influxdb.py [-v] [-f] <check_id>... [--influxdb=<influxdb>]

Options:
  -h --help              Show this screen
  -v                     Verbose
  -f                     Force plugin to send values as float
  <check_id>             Check id
  --influxdb=<influxdb>  influxdb DSN       [default: influxdb://<user>:<password>@localhost:8086/<database>]
"""

from docopt import docopt
import sys

from influxdb import InfluxDBClient

import logging
logging.basicConfig(
    format='%(asctime)s %(levelname)s %(name)s %(message)s',
    level=logging.INFO
)

logger = logging.getLogger("plugin")

logging.getLogger("requests").setLevel(logging.WARNING)

try:
    from urllib.parse import urlparse
except:
    # Python 2
    from urlparse import urlparse

try:
    from checkmyws import CheckmywsClient
except:
    logger.error("Error: please install Check my Website client 'pip install checkmyws-python'")
    sys.exit(1)


def influxdb_format(measurement, tags, value, timestamp):
    return {
        "tags": tags,
        "points": [{
            "measurement": measurement,
            "fields": {"value": value},
            "time": timestamp * 1000000000
        }]
    }


def influxdb_write(influxdb, metric):
    logger.debug("InfluxDB Write: %s", metric)

    try:
        influxdb.write(metric, params={'db': influxdb._database})

    except Exception as err:
        logger.error("InfluxDB Impossible to write data %s: %s", metric, err)


def get_data_from_cmws(check_id, asfloat=False):
    # Get data from Check my Website
    cmws = CheckmywsClient()
    raw = cmws.status(check_id)

    metrics = []

    # Format data
    url_raw = raw["url"]
    url = urlparse(url_raw)
    hostname = url.netloc.replace(":", ".")
    path = url.path

    if not path:
        path = "/"

    timestamp = raw["metas"]["lastcheck"]

    # Httptime by location
    for (location, value) in raw["lastvalues"]["httptime"].items():
        tags = {
            '_id': check_id,
            'url': url_raw,
            'hostname': hostname,
            'location': location,
            'path': path
        }

        if asfloat:
            value = float(asfloat)

        metric = influxdb_format(
            "httptime",
            tags,
            value,
            timestamp
        )

        metrics.append(metric)

    # States by location
    for (location, value) in raw["states"].items():
        tags = {
            '_id': check_id,
            'url': url_raw,
            'hostname': hostname,
            'location': location,
            'path': path
        }

        if asfloat:
            value = float(asfloat)

        metric = influxdb_format(
            "state",
            tags,
            value,
            timestamp
        )

        metrics.append(metric)

    # Metas
    blacklist = (
        'title', 'lastcheck', 'laststatechange_bin', 'laststatechange',
        'dns_expiration_timestamp', 'ssl_cert_expiration_timestamp'
    )
    for (label, value) in raw["metas"].items():
        if label in blacklist:
            continue

        tags = {
            '_id': check_id,
            'url': url_raw,
            'hostname': hostname,
            'path': path
        }<commit_msg>Tweak: use base tags dict<commit_after>#!/usr/bin/env python
# -*- coding: utf8 -*-

"""Check my Website to InfluxDB

Usage:
  cmws2influxdb.py [-v] [-f] <check_id>... [--influxdb=<influxdb>]

Options:
  -h --help              Show this screen
  -v                     Verbose
  -f                     Force plugin to send values as float
  <check_id>             Check id
  --influxdb=<influxdb>  influxdb DSN       [default: influxdb://<user>:<password>@localhost:8086/<database>]
"""

from docopt import docopt
import sys

from influxdb import InfluxDBClient

import logging
logging.basicConfig(
    format='%(asctime)s %(levelname)s %(name)s %(message)s',
    level=logging.INFO
)

logger = logging.getLogger("plugin")

logging.getLogger("requests").setLevel(logging.WARNING)

try:
    from urllib.parse import urlparse
except:
    # Python 2
    from urlparse import urlparse

try:
    from checkmyws import CheckmywsClient
except ImportError as err:
    logger.error("Error: please install Check my Website client 'pip install checkmyws-python'")
    sys.exit(1)


def influxdb_format(measurement, tags, value, timestamp):
    return {
        "tags": tags,
        "points": [{
            "measurement": measurement,
            "fields": {"value": value},
            "time": timestamp * 1000000000
        }]
    }


def influxdb_write(influxdb, metric):
    logger.debug("InfluxDB Write: %s", metric)

    try:
        influxdb.write(metric, params={'db': influxdb._database})

    except Exception as err:
        logger.error("InfluxDB Impossible to write data %s: %s", metric, err)


def get_data_from_cmws(check_id, asfloat=False):
    # Get data from Check my Website
    cmws = CheckmywsClient()
    raw = cmws.status(check_id)

    metrics = []

    # Format data
    url_raw = raw["url"]
    url = urlparse(url_raw)
    hostname = url.netloc.replace(":", ".")
    path = url.path

    if not path:
        path = "/"

    timestamp = raw["metas"]["lastcheck"]

    tags = {
        '_id': check_id,
        'url': url_raw,
        'hostname': hostname,
        'path': path
    }

    # Httptime by location
    for (location, value) in raw["lastvalues"]["httptime"].items():
        t = dict(tags)
        t['location'] = location

        if asfloat:
            value = float(asfloat)

        metric = influxdb_format(
            "httptime",
            t,
            value,
            timestamp
        )

        metrics.append(metric)

    # States by location
    for (location, value) in raw["states"].items():
        t = dict(tags)
        t['location'] = location

        if asfloat:
            value = float(asfloat)

        metric = influxdb_format(
            "state",
            t,
            value,
            timestamp
        )

        metrics.append(metric)

    # Metas
    blacklist = (
        'title', 'lastcheck', 'laststatechange_bin', 'laststatechange',
        'dns_expiration_timestamp', 'ssl_cert_expiration_timestamp'
    )
    for (label, value) in raw["metas"].items():
        if label in blacklist:
            continue
{'fraction_non_alphanumeric': 0.12123576220939304, 'fraction_numerical': 0.0056171009517865505, 'mean_word_length': 2.1086323957322985, 'pattern_counts': {'":': 14, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2455211', 'n_tokens_mistral': 2015, 'n_tokens_neox': 1929, 'n_words': 487}