<commit_before>#!/usr/bin/env python

#author <NAME>
#licence : apache v2


from pymongo import MongoClient
import time

import re

#psutil to perform system command
import psutil

# for ceph command call
import subprocess

import datetime

import sys
import os
import getopt
import socket
from daemon import Daemon
 
import json
from StringIO import StringIO

from bson.dbref import DBRef 

from threading import Thread, Event

import httplib

import signal

#from bson.objectid import ObjectId
#db.col.find({"_id": ObjectId(obj_id_to_find)})

configfile = "/opt/inkscope/etc/cephprobe.conf"
runfile = "/var/run/cephprobe/cephprobe.pid"
logfile = "/var/log/cephprobe.log"
clusterName = "ceph"
fsid = ""




#load the conf (from json into file)
def load_conf():
    datasource = open(configfile, "r")
    data = json.load(datasource)
    
    datasource.close()
    return data


# list sections prefixed
def ceph_conf_list(prefix):
    p = subprocess.Popen(
        args=[
            'ceph-conf',
            '-l',
            prefix
            ],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)
    outdata, errdata = p.communicate()
    if (len(errdata)):
        raise RuntimeError('unable to get conf option prefix %s: %s' % (prefix, errdata))
    return outdata.rstrip().splitlines();


# get a field value from named section
def ceph_conf(field, name):
    p = subprocess.Popen(
        args=[
            'ceph-conf',
            '--show-config-value',
            field,
            '-n',
            name,
            ],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)
    outdata, errdata = p.communicate()
    if (len(errdata)):
        raise RuntimeError('unable to get conf option %s for %s: %s' % (field, name, errdata))
    return outdata.rstrip()


# get a field value from global conf
def ceph_conf_global(field):
    p = subprocess.Popen(
        args=[
            'ceph-conf',
            '--show-config-value',
            field
            ],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)
    outdata, errdata = p.communicate()
    if (len(errdata)):
        raise RuntimeError('unable to get conf option %s: %s' % (field, errdata))
    return outdata.rstrip()


# extract mons from conf and put them into mons
def processConf():
    mon_sections=ceph_conf_list('mon.')
    if (len(mon_sections)==0):
        initmon = ceph_conf_global('mon_initial_members')
        if (not initmon):
            raise RuntimeError('enable to find a mon')
        mons = [initmon]
    else:
        for mon in mon_sections:
            mons.append(ceph_conf('host', mon))
        




#cluster
def initCluster(restapi, db):
    global fsid
    fsid = processStatus(restapi, db)
    processCrushmap(restapi, db)
    processOsdDump(restapi, db)
    processPgDump(restapi, db)
    processDf(restapi, db)
    
    
    


#uri : /api/v0.1/status.json
def processStatus(restapi, db):
    print str(datetime.datetime.now()), "-- Process Status"  
    sys.stdout.flush()
    restapi.request("GET", "/api/v0.1/status.json")
    r1=restapi.getresponse()
    if (r1.status == 200) :
        data1 = r1.read()
        io = StringIO(data1)
        c_status = json.load(io)
        
        monmap = c_status['output']['monmap']
        
        map_stat_mon = {}
        health_services_list = c_status['output']['health']['health']['health_services']
        
        for health_service in health_services_list:
            health_services_mons = health_service['mons']        
            for monst in health_services_mons:
                monstat = monst.copy()
                monstat["mon"] =  DBRef( "mon", monst['name'])
                monstat["_id"] = monst['name']+":"+monst["last_updated"]
                del monstat["name"]
                db.monstat.update({"_id" : monstat["_id"]}, monstat, upsert= True)
                map_stat_mon[monst['name']] = monstat["_id"]
        
        map_rk_name = {}
        
        for mon in monmap['mons'] :
            mondb = {"_id" : mon['name'],
               "host" : DBRef( "hosts", mon['name']),
               "addr" : mon['addr'],
               "rank" : mon['rank'],
               "stat" : DBRef("monstat", map_stat_mon[mon['name']])
               }
            db.mon.update({"_id" : mon['name']}, mondb, upsert= True)
            map_rk_name[mon['rank']] =  mon['name']        
            #no skew and latency ? 
            
        mm = {"epoch" : monmap['epoch'],
              "created" : monmap['created'],
              "modified" : monmap['modified'],
              "mons" : [DBRef( "mon", m['name']) for m in monmap['mons']],
              "quorum" :  [DBRef( "mon", map_rk_name[rk]) for rk in c_status['output']['quorum']]
              }      
        cluster = {"_id" : c_status['output']['fsid'], 
                   "election_epoch" : c_status['output']['election_epoch'], 
                   "monmap" : mm,
                   "pgmap" : c_status['output']['pgmap'],
                   "osdmap-info" : c_status['output']['osdmap']['osdmap'],
                   "name" : clusterName, 
                   "health" : c_status['output']['health']['overall_status']
                   }     
        db.cluster.update({'_id' : c_status['output']['fsid']}, cluster, upsert= True)
        
        return c_status['output']['fsid']
   

#uri : /api/v0.1/osd/dump.json
def processOsdDump(restapi, db):   
    print str(datetime.datetime.now()), "-- Process OSDDump"  
    sys.stdout.flush()
    restapi.request("GET", "/api/v0.1/osd/dump.json")<commit_msg>add timecheck and health description<commit_after>#!/usr/bin/env python

#author <NAME>
#licence : apache v2


from pymongo import MongoClient
import time

import re

#psutil to perform system command
import psutil

# for ceph command call
import subprocess

import datetime

import sys
import os
import getopt
import socket
from daemon import Daemon
 
import json
from StringIO import StringIO

from bson.dbref import DBRef 

from threading import Thread, Event

import httplib

import signal

#from bson.objectid import ObjectId
#db.col.find({"_id": ObjectId(obj_id_to_find)})

configfile = "/opt/inkscope/etc/cephprobe.conf"
runfile = "/var/run/cephprobe/cephprobe.pid"
logfile = "/var/log/cephprobe.log"
clusterName = "ceph"
fsid = ""




#load the conf (from json into file)
def load_conf():
    datasource = open(configfile, "r")
    data = json.load(datasource)
    
    datasource.close()
    return data


# list sections prefixed
def ceph_conf_list(prefix):
    p = subprocess.Popen(
        args=[
            'ceph-conf',
            '-l',
            prefix
            ],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)
    outdata, errdata = p.communicate()
    if (len(errdata)):
        raise RuntimeError('unable to get conf option prefix %s: %s' % (prefix, errdata))
    return outdata.rstrip().splitlines();


# get a field value from named section
def ceph_conf(field, name):
    p = subprocess.Popen(
        args=[
            'ceph-conf',
            '--show-config-value',
            field,
            '-n',
            name,
            ],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)
    outdata, errdata = p.communicate()
    if (len(errdata)):
        raise RuntimeError('unable to get conf option %s for %s: %s' % (field, name, errdata))
    return outdata.rstrip()


# get a field value from global conf
def ceph_conf_global(field):
    p = subprocess.Popen(
        args=[
            'ceph-conf',
            '--show-config-value',
            field
            ],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)
    outdata, errdata = p.communicate()
    if (len(errdata)):
        raise RuntimeError('unable to get conf option %s: %s' % (field, errdata))
    return outdata.rstrip()


# extract mons from conf and put them into mons
def processConf():
    mon_sections=ceph_conf_list('mon.')
    if (len(mon_sections)==0):
        initmon = ceph_conf_global('mon_initial_members')
        if (not initmon):
            raise RuntimeError('enable to find a mon')
        mons = [initmon]
    else:
        for mon in mon_sections:
            mons.append(ceph_conf('host', mon))
        




#cluster
def initCluster(restapi, db):
    global fsid
    fsid = processStatus(restapi, db)
    processCrushmap(restapi, db)
    processOsdDump(restapi, db)
    processPgDump(restapi, db)
    processDf(restapi, db)
   
#health value 
healthCst = ["HEALTH_OK", "HEALTH_WARN", "HEALTH_ERROR"]
healthMap = {}
for idx, h in enumerate(healthCst) :
    healthMap[h] = idx
    
    
def worstHealth(h1, h2) :
    return healthCst[max(healthMap[h1], healthMap[h2])]

#uri : /api/v0.1/status.json
def processStatus(restapi, db):
    print str(datetime.datetime.now()), "-- Process Status"  
    sys.stdout.flush()
    restapi.request("GET", "/api/v0.1/status.json")
    r1=restapi.getresponse()
    if (r1.status == 200) :
        data1 = r1.read()
        io = StringIO(data1)
        c_status = json.load(io)
        
        monmap = c_status['output']['monmap']
        
        map_stat_mon = {}
        health_services_list = c_status['output']['health']['health']['health_services']
       
        time_checks  = c_status['output']['health']['timechecks']
        timecheckmap = {}
        for tc in time_checks["mons"]:
            tc["time_health"] = tc["health"]
            del tc["health"]
            monname = tc["name"]
            del tc["name"]
            timecheckmap[monname] = tc
            
        
        
        # complete timecheck
        
        for health_service in health_services_list:
            health_services_mons = health_service['mons']        
            for monst in health_services_mons:
                monstat = monst.copy()
                monstat["mon"] =  DBRef( "mon", monst['name'])
                monstat["_id"] = monst['name']+":"+monst["last_updated"]
                monstat["capacity_health"] = monstat["health"] 
                
                #complete with timecheck
                tc = timecheckmap[monstat["name"]]
                monstat.update(tc)
                
                monstat["health"] = worstHealth(monstat["capacity_health"], monstat["time_health"] )
                del monstat["name"]
                db.monstat.update({"_id" : monstat["_id"]}, monstat, upsert= True)
                map_stat_mon[monst['name']] = monstat["_id"]
        
        map_rk_name = {}
        
        for mon in monmap['mons'] :
            mondb = {"_id" : mon['name'],
               "host" : DBRef( "hosts", mon['name']),
               "addr" : mon['addr'],
               "rank" : mon['rank'],
               "stat" : DBRef("monstat", map_stat_mon[mon['name']])
               }
            db.mon.update({"_id" : mon['name']}, mondb, upsert= True)
            map_rk_name[mon['rank']] =  mon['name']        
            #no skew and latency ? 
            
        mm = {"epoch" : monmap['epoch'],
              "created" : monmap['created'],
              "modified" : monmap['modified'],
              "mons" : [DBRef( "mon", m['name']) for m in monmap['mons']],
              "quorum" :  [DBRef( "mon", map_rk_name[rk]) for rk in c_status['output']['quorum']]
              }      
        cluster = {"_id" : c_status['output']['fsid'], 
                   "election_epoch" : c_status['output']['election_epoch'], 
                   "monmap" : mm,
                   "pgmap" : c_status['output']['pgmap'],
                   "osdmap-info" : c_status['output']['osdmap']['osdmap'],
                   "name" : clusterName, 
                   "health" : c_status['output']['health']['overall_status'],
                   "health_detail" : c_status['output']['health']['detail'],
                   "health_summary" : c_status['output']['health']['summary']
                   }     
        db.cluster.update({'_id' : c_status['output']['fsid']}, cluster, upsert= True)
        
        return c_status['output']['fsid']
   

#uri : /api/v0.1/osd/dump.json
def processOsdDump(restapi, db):   
    print str(datetime.datetime.now()), "-- Process OSDDump"  
    sys.stdout.flush()
    restapi.request("GET", "/api/v0.1/osd/dump.json")
{'fraction_non_alphanumeric': 0.13889342255590012, 'fraction_numerical': 0.003264240248082259, 'mean_word_length': 2.1691233514352213, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7119019', 'n_tokens_mistral': 3940, 'n_tokens_neox': 3825, 'n_words': 837}