<commit_before>#!/usr/bin/python
#
# nodewatcher monitoring daemon
#
# Copyright (C) 2009 by <NAME> <<EMAIL>>
#

# First parse options (this must be done here since they contain import paths
# that must be parsed before Django models can be imported)
import sys, os
from optparse import OptionParser

print "============================================================================"
print "                       nodewatcher monitoring daemon"
print "============================================================================"

parser = OptionParser()
parser.add_option('--path', dest = 'path', help = 'Path that contains nodewatcher "web" Python module')
parser.add_option('--settings', dest = 'settings', help = 'Django settings to use')
parser.add_option('--olsr-host', dest = 'olsr_host', help = 'A host with OLSR txt-info plugin running (overrides settings file)')
parser.add_option('--regenerate-graphs', dest = 'regenerate_graphs', help = 'Just regenerate graphs from RRAs and exit (only graphs that have the redraw flag set are regenerated)', action = 'store_true')
parser.add_option('--stress-test', dest = 'stress_test', help = 'Perform a stress test (only used for development)', action = 'store_true')
parser.add_option('--collect-simulation', dest = 'collect_sim', help = 'Collect simulation data', action = 'store_true')
parser.add_option('--update-rrds', dest = 'update_rrds', help = 'Update RRDs', action = 'store_true')
parser.add_option('--update-rrd-type', dest = 'update_rrd_type', help = 'Update RRD type (refresh, archive)', default = 'refresh')
parser.add_option('--reverse-populate', dest = 'reverse_populate', help = 'Reverse populate RRD with data from a database', action = 'store_true')
parser.add_option('--reverse-populate-node', dest = 'rp_node', help = 'Node to populate data for')
parser.add_option('--reverse-populate-graph', dest = 'rp_graph', help = 'Graph type to populate data for')
options, args = parser.parse_args()

if not options.path:
  print "ERROR: Path specification is required!\n"
  parser.print_help()
  exit(1)
elif not options.settings:
  print "ERROR: Settings specification is required!\n"
  parser.print_help()
  exit(1)
elif options.reverse_populate and (not options.rp_node or not options.rp_graph):
  print "ERROR: Reverse populate requires node and graph type!\n"
  parser.print_help()
  exit(1)

# Setup import paths, since we are using Django models
sys.path.append(options.path)
os.environ['DJANGO_SETTINGS_MODULE'] = options.settings

# Import our models
from web.nodes.models import Node, NodeStatus, Subnet, SubnetStatus, APClient, Link, GraphType, GraphItem, Event, EventSource, EventCode, IfaceType, InstalledPackage, NodeType, RenumberNotice, WarningCode, NodeWarning, Tweet
from web.generator.models import Template, Profile
from django.db import transaction, models, connection
from django.conf import settings

# Possibly override MONITOR_OLSR_HOST setting with comomand line option
if options.olsr_host:
  settings.MONITOR_OLSR_HOST = options.olsr_host

# Import other stuff
if getattr(settings, 'MONITOR_ENABLE_SIMULATION', None) or options.stress_test:
  from simulator import nodewatcher, wifi_utils
else:
  from lib import nodewatcher, wifi_utils
  
  # Setup simulation data collection
  nodewatcher.COLLECT_SIMULATION_DATA = options.collect_sim
  wifi_utils.COLLECT_SIMULATION_DATA = options.collect_sim

from lib.rra import RRA, RRAIface, RRAClients, RRARTT, RRALinkQuality, RRASolar, RRALoadAverage, RRANumProc, RRAMemUsage, RRALocalTraffic, RRANodesByStatus, RRAWifiCells, RRAOlsrPeers, RRAPacketLoss
from lib.topology import DotTopologyPlotter
from lib.local_stats import fetch_traffic_statistics
from lib import ipcalc
from time import sleep
from datetime import datetime, timedelta
from traceback import format_exc, print_exc
import pwd
import logging
import time
import multiprocessing
import gc

if Tweet.tweets_enabled():
  from lib import bitly

RRA_CONF_MAP = {
  GraphType.RTT         : RRARTT,
  GraphType.LQ          : RRALinkQuality,
  GraphType.Clients     : RRAClients,
  GraphType.Traffic     : RRAIface,
  GraphType.LoadAverage : RRALoadAverage,
  GraphType.NumProc     : RRANumProc,
  GraphType.MemUsage    : RRAMemUsage,
  GraphType.Solar       : RRASolar,
  GraphType.WifiCells   : RRAWifiCells,
  GraphType.OlsrPeers   : RRAOlsrPeers,
  GraphType.PacketLoss  : RRAPacketLoss
}
WORKER_POOL = None

def safe_int_convert(integer):
  """
  A helper method for converting a string to an integer.
  """
  try:
    return int(integer)
  except:
    return None

def safe_loadavg_convert(loadavg):
  """
  A helper method for converting a string to a loadavg tuple.
  """
  try:
    loadavg = loadavg.split(' ')
    la1min, la5min, la15min = (float(x) for x in loadavg[0:3])
    nproc = int(loadavg[3].split('/')[1])
    return la1min, la5min, la15min, nproc
  except:
    return None, None, None, None

def safe_uptime_convert(uptime):
  """
  A helper method for converting a string to an uptime integer.
  """
  try:
    return int(float(uptime.split(' ')[0]))
  except:
    return None

def safe_date_convert(timestamp):
  """
  A helper method for converting a string timestamp into a datetime
  object.
  """
  try:
    return datetime.fromtimestamp(int(timestamp))
  except:
    return None

def add_graph(node, name, type, conf, title, filename, *values, **attrs):
  """
  A helper function for generating graphs.
  """
  if getattr(settings, 'MONITOR_DISABLE_GRAPHS', None):
    return
  
  # Get parent instance (toplevel by default)
  parent = attrs.get('parent', None)

  try:
    graph = GraphItem.objects.get(node = node, name = name, type = type, parent = parent)
  except GraphItem.DoesNotExist:
    graph = GraphItem(node = node, name = name, type = type, parent = parent)
    graph.rra = '%s_%s.rrd' % (filename, node.pk)
    graph.graph = '%s_%s.png' % (filename, node.pk)
  
  rra = str(os.path.join(settings.MONITOR_WORKDIR, 'rra', graph.rra))
  try:
    RRA.update(node, conf, rra, *values, graph = graph.pk)
  except:
    logging.warning(format_exc())
  
  graph.title = title
  graph.last_update = datetime.now()
  graph.dead = False
  graph.need_redraw = True
  graph.save()
  return graph

@transaction.commit_on_success
def check_events():
  """<commit_msg>Rename graph images on demand as per ticket #709.<commit_after>#!/usr/bin/python
#
# nodewatcher monitoring daemon
#
# Copyright (C) 2009 by <NAME> <<EMAIL>>
#

# First parse options (this must be done here since they contain import paths
# that must be parsed before Django models can be imported)
import sys, os
from optparse import OptionParser

print "============================================================================"
print "                       nodewatcher monitoring daemon"
print "============================================================================"

parser = OptionParser()
parser.add_option('--path', dest = 'path', help = 'Path that contains nodewatcher "web" Python module')
parser.add_option('--settings', dest = 'settings', help = 'Django settings to use')
parser.add_option('--olsr-host', dest = 'olsr_host', help = 'A host with OLSR txt-info plugin running (overrides settings file)')
parser.add_option('--regenerate-graphs', dest = 'regenerate_graphs', help = 'Just regenerate graphs from RRAs and exit (only graphs that have the redraw flag set are regenerated)', action = 'store_true')
parser.add_option('--stress-test', dest = 'stress_test', help = 'Perform a stress test (only used for development)', action = 'store_true')
parser.add_option('--collect-simulation', dest = 'collect_sim', help = 'Collect simulation data', action = 'store_true')
parser.add_option('--update-rrds', dest = 'update_rrds', help = 'Update RRDs', action = 'store_true')
parser.add_option('--update-rrd-type', dest = 'update_rrd_type', help = 'Update RRD type (refresh, archive)', default = 'refresh')
parser.add_option('--reverse-populate', dest = 'reverse_populate', help = 'Reverse populate RRD with data from a database', action = 'store_true')
parser.add_option('--reverse-populate-node', dest = 'rp_node', help = 'Node to populate data for')
parser.add_option('--reverse-populate-graph', dest = 'rp_graph', help = 'Graph type to populate data for')
options, args = parser.parse_args()

if not options.path:
  print "ERROR: Path specification is required!\n"
  parser.print_help()
  exit(1)
elif not options.settings:
  print "ERROR: Settings specification is required!\n"
  parser.print_help()
  exit(1)
elif options.reverse_populate and (not options.rp_node or not options.rp_graph):
  print "ERROR: Reverse populate requires node and graph type!\n"
  parser.print_help()
  exit(1)

# Setup import paths, since we are using Django models
sys.path.append(options.path)
os.environ['DJANGO_SETTINGS_MODULE'] = options.settings

# Import our models
from web.nodes.models import Node, NodeStatus, Subnet, SubnetStatus, APClient, Link, GraphType, GraphItem, Event, EventSource, EventCode, IfaceType, InstalledPackage, NodeType, RenumberNotice, WarningCode, NodeWarning, Tweet
from web.generator.models import Template, Profile
from django.db import transaction, models, connection
from django.conf import settings

# Possibly override MONITOR_OLSR_HOST setting with comomand line option
if options.olsr_host:
  settings.MONITOR_OLSR_HOST = options.olsr_host

# Import other stuff
if getattr(settings, 'MONITOR_ENABLE_SIMULATION', None) or options.stress_test:
  from simulator import nodewatcher, wifi_utils
else:
  from lib import nodewatcher, wifi_utils
  
  # Setup simulation data collection
  nodewatcher.COLLECT_SIMULATION_DATA = options.collect_sim
  wifi_utils.COLLECT_SIMULATION_DATA = options.collect_sim

from lib.rra import RRA, RRAIface, RRAClients, RRARTT, RRALinkQuality, RRASolar, RRALoadAverage, RRANumProc, RRAMemUsage, RRALocalTraffic, RRANodesByStatus, RRAWifiCells, RRAOlsrPeers, RRAPacketLoss
from lib.topology import DotTopologyPlotter
from lib.local_stats import fetch_traffic_statistics
from lib import ipcalc
from time import sleep
from datetime import datetime, timedelta
from traceback import format_exc, print_exc
import pwd
import logging
import time
import multiprocessing
import gc

if Tweet.tweets_enabled():
  from lib import bitly

RRA_CONF_MAP = {
  GraphType.RTT         : RRARTT,
  GraphType.LQ          : RRALinkQuality,
  GraphType.Clients     : RRAClients,
  GraphType.Traffic     : RRAIface,
  GraphType.LoadAverage : RRALoadAverage,
  GraphType.NumProc     : RRANumProc,
  GraphType.MemUsage    : RRAMemUsage,
  GraphType.Solar       : RRASolar,
  GraphType.WifiCells   : RRAWifiCells,
  GraphType.OlsrPeers   : RRAOlsrPeers,
  GraphType.PacketLoss  : RRAPacketLoss
}
WORKER_POOL = None

def safe_int_convert(integer):
  """
  A helper method for converting a string to an integer.
  """
  try:
    return int(integer)
  except:
    return None

def safe_loadavg_convert(loadavg):
  """
  A helper method for converting a string to a loadavg tuple.
  """
  try:
    loadavg = loadavg.split(' ')
    la1min, la5min, la15min = (float(x) for x in loadavg[0:3])
    nproc = int(loadavg[3].split('/')[1])
    return la1min, la5min, la15min, nproc
  except:
    return None, None, None, None

def safe_uptime_convert(uptime):
  """
  A helper method for converting a string to an uptime integer.
  """
  try:
    return int(float(uptime.split(' ')[0]))
  except:
    return None

def safe_date_convert(timestamp):
  """
  A helper method for converting a string timestamp into a datetime
  object.
  """
  try:
    return datetime.fromtimestamp(int(timestamp))
  except:
    return None

def add_graph(node, name, type, conf, title, filename, *values, **attrs):
  """
  A helper function for generating graphs.
  """
  if getattr(settings, 'MONITOR_DISABLE_GRAPHS', None):
    return
  
  # Get parent instance (toplevel by default)
  parent = attrs.get('parent', None)
  graph_image_name = '%s_%s.png' % (filename, node.pk)
  graph_rra_name = '%s_%s.rrd' % (filename, node.pk)

  try:
    graph = GraphItem.objects.get(node = node, name = name, type = type, parent = parent)
    if graph.graph != graph_image_name:
      graph.graph = graph_image_name
  except GraphItem.DoesNotExist:
    graph = GraphItem(node = node, name = name, type = type, parent = parent)
    graph.rra = graph_rra_name
    graph.graph = graph_image_name
  
  rra = str(os.path.join(settings.MONITOR_WORKDIR, 'rra', graph.rra))
  try:
    RRA.update(node, conf, rra, *values, graph = graph.pk)
  except:
    logging.warning(format_exc())
  
  graph.title = title
  graph.last_update = datetime.now()
  graph.dead = False
  graph.need_redraw = True
  graph.save()
  return graph

@transaction.commit_on_success
def check_events():
  """
{'fraction_non_alphanumeric': 0.12753941485606715, 'fraction_numerical': 0.003372813554004236, 'mean_word_length': 4.664149266992448, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '752708', 'n_tokens_mistral': 4105, 'n_tokens_neox': 3962, 'n_words': 1173}