Gnmond Documentation

from HealthLogicFramework import * #import framework, has to be done!

def init():
        """the init() function
        will be called once on start-up"""

        """with setLoggin you can set the logging settings
        GnmondLogger.DEBUG means log everything.
        You might want to change this to GnmondLogger.WARNING
        """

        setLogging(GnmondLogger.DEBUG)

        """with setExecutionTimeInterval you can set the execution Interval 
        of the analyze function
        analyze is called every executionIntervall minutes
        If set to zero analyze is called every minute
        """
        setExecutionInterval(2)

        """with setMaxExecutionTime you can set the  max. execution time of analyze()
        in sec. If analyzes takes longer than this to finish, 
        it will be stopped and reported as invalid.
        This should prevent a hanging plug-in crashing the hole system
        """
        setMaxExecutionTime(1)


        """with add allowed server you can add a server to allowedServers list
        Only hosts in this list will be allowed to connect to gnmond, all other
        connections will be refused
        """

        addAllowedServer("localhost")

        """ with addCluster you can add a cluster that should be monitored.
        You should give this cluster a name and a list of initial hosts.
        If you use gmond input plug-in those names have to be exactly 
        the same as in ganglia!
        """
        addCluster("test",["pc0.psi.ch","pc1.psi.ch"])

        """with add record you can add a record
        You should give the record a name and a default status/message.
        """

        addRecord("test",NAGIOS_OK,"Everything fine")

def analyze(value):
        """with values.getMax("load_percent") you'll get 
        the maximum of the metric load_percent 
        of all nodes is clusters defined in the init() function.
        Note that load_percent is defined as the ganglia metrics load_one / num_cpu * 100
        """
        maxLoadInCluster = values.getMax("load_percent")
        if maxLoadInCluster > 200:

                """if one node has a load above 200, 
                you want to set record test to CRITICAL.
                This can be done with setRecord
                """
                setRecord("test",NAGIOS_CRITICAL,"Some nodes are overloaded")

        """If you won't set a record during analyze, 
       it will be reset to default values defined in addRecord.
       """"

def clusterFailure(cluster):
        """clusterFailure will be called every time a cluster cannot be reached.
        You might want to set your records to critical or unknown...
        This record could be overwritten by the analyze() function, 
        but only if analyze would "worsen" it.
        (e.g. clusterFailure sets it to warning, but analyze to critical,
        the record would result as critical)
        """
        setRecord("test",NAGIOS_CRITICAL,"Could not reach cluster")

from HealthLogicFramework import *

def init():
        addCluster("test",["pc0.psi.ch","pc1.psi.ch"])
        addGroup("testg",["pc0.psi.ch","pc4.psi.ch"])
        addRecord("test",NAGIOS_OK,"Everything fine")
        addRecord("test2",NAGIOS_OK,"Everything fine")

def analyze(value):
        maxLoadInCluster = values.getMax("load_percent")
        if maxLoadInCluster > 200:
                setRecord("test",NAGIOS_CRITICAL,"Some nodes are overloaded")

        """with getGroup() you'll get a metric object 
        for the group defined in the init() function.
        This object is of the same type as value and allowes the same calls
        """
        maxLoadInGroup = getGroup("testg").getMax("load_percent")
        if maxLoadInGroup > 100:
                setRecord("testg",NAGIOS_WARNING,"Some nodes in group are overloaded")

def clusterFailure(cluster):
        setRecord("test",NAGIOS_CRITICAL,"Could not reach cluster")
        setRecord("testg",NAGIOS_CRITICAL,"Could not reach cluster")

from HealthLogicFramework import *

def init():
        addCluster("test",["pc0.psi.ch","pc1.psi.ch"])
        addRecord("test_last",NAGIOS_OK,"Everything fine")
        addRecord("test_period",NAGIOS_OK,"Everything fine")

        """with store() you can mark a metric to be stored. This can be called on 
        every metric object you get with getNode, getCluster, getGroup.
        You have to set the size of the storage
        """
        getCluster("test").store("load_percent",15)

def analyze(value):

        c = getCluster("test")

        """with fetch() you can access a stored value.
        if called without argument or 0 it will return the actual value, with an
        int as argument it will return the metric with this age
        """
        if c.fetch(1,"load_percent") > 200:
                setRecord("test",NAGIOS_WARNING,"Some nodes were overloaded\
                last time")

        below = False
        for metric in c.fetchAll("load_percent"):
                if metric < 200:
                        below = True
                        break
        if not below:
                setRecord("test_longer",NAGIOS_WARNING,"For history load was\
                above 200%")

def clusterFailure(cluster):
        setRecord("test_last",NAGIOS_CRITICAL,"Could not reach cluster")
        setRecord("test_period",NAGIOS_CRITICAL,"Could not reach cluster")

from HealthLogicFramework import *
from DefaultChecks import * # To use default check you have to import this

cluster_name = "test"

#length of history that default testes will check
length_of_history = 30

#Number of nodes that can fail the test until critical
tolerated = 3

#nodes in cluster. For default_checks the nodes has to be know in preview
nodes = ["pc0.psi.ch","pc1.psi.ch","pc2.psi.ch","pc3.psi.ch","pc4.psi.ch"]

def init():
def init():
        addCluster(cluster_name,nodes)
        #Sets up all default checks as described in DefaultChecks
        setUpAll(cluster_name,length_of_history)

def analyze(value):
        checkAll(clusterName,tolerated) #Perfore all checks

def clusterFailure(cluster):
        pass #Sine no records are defined, you have to do nothing...

from HealthLogicFramework import *
from DefaultChecks import *


#Unlike example 4 you have to sort the nodes in 3 Groups
login_nodes=["testlogin.psi.ch"]
file_nodes=["testfiles.psi.ch","testfiles2.psi.ch"]
compute_nodes =["pc0.psi.ch","pc1.psi.ch","pc2.psi.ch","pc3.psi.ch","pc4.psi.ch"]

clusterName = "test"
tolerated = 5
history=15

def init():
        addCluster(clusterName,compute_nodes)

        """set up the singel check as described in DefaultChecks
        """
        setUpSingleCheck(getCluster(clusterName),login_nodes\
        ,file_nodes,compute_nodes)

def analyze(value):
        singleCheck(getCluster(clusterName)) #perfor single check

def clusterFailure(cluster):
        """singleCheck defines the record 'clusterName' you might want to set this
        record to CRITICAL or UNKNOWN if cluster cannot be reached...
        """
        setRecord(clusterName,NAGIOS_CRITICAL,clusterName + " could not be reached")

from HealthLogicFramework import *
import Plugins.Input.YOURPLUGIN as Input #Import your plug-in

def init():
        #Give your plug-in to addCluster
        addCluster("test",["pc0.psi.ch","pc1.psi.ch"], Input) 
        addRecord("test",NAGIOS_OK,"Everything fine")

def analyze(value):
        #Assusmes your input plug-in returns a metric names load_percent
	maxLoadInCluster = values.getMax("load_percent")
        if maxLoadInCluster > 200:
                setRecord("test",NAGIOS_CRITICAL,"Some nodes are overloaded")

def clusterFailure(cluster):
        setRecord("test",NAGIOS_CRITICAL,"Could not reach cluster")

"""
An example for a Output plug-in
"""

import threading
import copy

#The global variable records consists all records computed by health plugins
from HealthPluginManager import records

def getThread(serverList,logger):
        #Do not change this
        return OutputThread(serverList,logger)

def getName():
        #Change NAME to the name of your plug-in
        return "NAME"

class OutputThread (threading.Thread):
    def __init__(self,serverList,logger,):
        threading.Thread.__init__(self)
        self.serverList = copy.copy(serverList)
        self.logger = logger
        #Add stuff that should be set up before thread will be started here

    def run(self):
        while True:
            """Add your code here
            You can access global records to get access to records
            You can access self.serverList to get access to a list of servers, 
            who are allowed to connect to gnmond
            You can access self.logger to do some logging 
            (don't use print, since gnmond will normally run as daemon, 
            therefore those statements will be ignored).
            Get get more information about GnmondLogger see class GnmondLogger.
            """
            pass

    def terminate(self):
        """This code will be executed if gnmond terminates.
        For example you could give back some resources like sockets etc.
        """
        pass

"""
An example for a Input plug-in.
"""

maximalExecutionTime = 1 #The maximal execution time of getMetrics()

def getMetrics(cluster):
    """This function will be executed every time you want to get new values. 
    The argument is an object is of type HealthPluginManager.Cluster.
    With this element you have access to all needed stuff for example 
    (for more details see HealthPluginManager.Cluster)
        cluster.logger:   A GnmondLogger, used for logging output
        cluster.name:     Name of the cluster
        cluster.nodes:    A list of nodes in this cluster
                          IMPORTANT: This list might not be complete, 
                          especially in the first call.
                          The Input Plug-in shout update this list!
        cluster.nextNodesToCheck:
                          A List of nodes that should be checked next.
                          IMPORTANT: You have to manage this list by yourself!
        cluster.values:   Actual metrics of this cluster.
                          Your plug-in should set new metrics in this variable
                          values is a double dict with structure
                          {"node1":{"metric1":v1,"metric2":v2,...},"node2":{...},...}
                          The metrics itself should be floats.
        """

        """set the nodes in this cluster. IMPORTANT this should not be done statical.
        Set the nodes is actually only important for that first call of the plug-in,
        since you do not have to define all nodes in the health plugins init() function,
        but the analyze() function might has to have a list of nodes.
        """
        cluster.nodes = ["pc0.psi.ch","pc1.psi.ch","pc2.psi.ch","pc3.psi.ch","pc4.psi.ch"]

        """set the values for each node
        """
        cluster.values = dict()
        for node in cluster.nodes:
                cluster[node] = dict()
                cluster[node]["metric1"] = 0
                cluster[node]["metric2"] = 0
                cluster[node]["metric3"] = 0
                cluster[node]["metric4"] = 0
                cluster[node]["metric5"] = 0

define command {
    command_name    check_gnmond
    command_line    $USER5$/check_gnmond $HOSTADDRESS$ $ARG1$
}

define host{
        use                     generic-host
        host_name               GnmondServer
        alias                   SERVERNAME
        address                 129.129.194.94
        check_command           check-host-alive
        max_check_attempts      10
        contact_groups          linux-admins
        notification_interval   480
        notification_period     24x7
        notification_options    d,u
        }

define service{
        use                             generic-service
        host_name                       GnmondServer
        service_description             clusters
        is_volatile                     0
        check_period                    24x7
        max_check_attempts              1
        normal_check_interval           1
        retry_check_interval            1
        contact_groups                  linux-admins
        notification_interval           240
        notification_period             24x7
        notification_options            w,u,c
        check_command                   check_gnmond!gnmond_clusters
       }
define service{
        use                             generic-service
        host_name                       GnmondServer
        service_description             plugins
        is_volatile                     0
        check_period                    24x7
        max_check_attempts              1
        normal_check_interval           1
        retry_check_interval            1
        contact_groups                  linux-admins
        notification_interval           240
        notification_period             24x7
        notification_options            w,u,c
        check_command                   check_gnmond!gnmond_healthPlugins
       }

define service{
        use                             generic-service
        host_name                       GnmondServer
        service_description             RECORD_NAME
        is_volatile                     0
        check_period                    24x7
        max_check_attempts              1
        normal_check_interval           1
        retry_check_interval            1
        contact_groups                  linux-admins
        notification_interval           240
        notification_period             24x7
        notification_options            w,u,c
        check_command                   check_gnmond!RECORD_NAME
       }