Origo OS

#!/usr/bin/perl

#

# mon - schedules service tests and triggers alerts upon failures

#

# Jim Trocki, trockij@arctic.org

#

# $Id: mon.pl,v 1.1 2012-10-23 19:57:32 cabo Exp $

#

# Copyright (C) 1998 Jim Trocki

#

#    This program is free software; you can redistribute it and/or modify

#    it under the terms of the GNU General Public License as published by

#    the Free Software Foundation; either version 2 of the License, or

#    (at your option) any later version.

#

#    This program is distributed in the hope that it will be useful,

#    but WITHOUT ANY WARRANTY; without even the implied warranty of

#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

#    GNU General Public License for more details.

#

#    You should have received a copy of the GNU General Public License

#    along with this program; if not, write to the Free Software

#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

#

#

use strict;

my $RCSID='$Id: mon.pl,v 1.1 2012-10-23 19:57:32 cabo Exp $';

my $AUTHOR='trockij@arctic.org';

my $RELEASE='$Name:  $';

#

# NetBSD rc.d script compatibility

#

$0= "mon" . " " . join(" ", @ARGV) if $^O eq "netbsd";

#

# modules in the perl distribution

#

use Getopt::Long qw(:config no_ignore_case);

use Text::ParseWords;

use POSIX;

use Fcntl;

use Socket;

use Sys::Hostname;

use Sys::Syslog qw(:DEFAULT);

use FileHandle;

use Data::Dumper;

#

# CPAN modules

#

use Time::HiRes qw(gettimeofday tv_interval usleep);

use Time::Period;

sub auth;

sub call_alert;

sub check_auth;

sub clear_timers;

sub client_accept;

sub client_close;

sub client_command;

sub client_dopending;

sub client_write_opstatus;

sub collect_output;

sub daemon;

sub debug;

sub debug_dir;

sub dep_ok;

sub dep_summary;

sub depend;

sub dhmstos;

sub die_die;

sub disen_host;

sub disen_service;

sub disen_watch;

sub do_alert;

sub do_startup_alerts;

sub err_startup;

sub esc_str;

sub gen_scriptdir_hash;

sub handle_io;

sub handle_trap;

sub handle_trap_timeout;

sub host_exists;

sub host_singleton_group;

sub inRange;

sub init_cf_globals;

sub init_globals;

sub load_auth;

sub load_state;

sub normalize_paths;

sub mysystem;

sub init_dtlog;

sub pam_conv_func;

sub proc_cleanup;

sub process_event;

sub randomize_startdelay;

sub read_cf;

sub readhistoricfile;

sub reload;

sub remove_proc;

sub reset_server;

sub run_monitor;

sub save_state;

sub set_last_test;

sub set_op_status;

sub reset_timer;

sub setup_server;

sub sock_write;

sub syslog_die;

sub un_esc_str;

sub usage;

sub write_dtlog;

#

# globals

#

my %opt;		# cmdline arguments

my %CF;			# configuration directives

my $PWD;		# current working directory

my $HOSTNAME;		# system hostname

my $STOPPED;		# 1 = scheduler stopped, 0 = not stopped

my $STOPPED_TIME;	# time(2) scheduler was stopped, if stopped

my $SLEEPINT;		# don't touch

my %watch_disabled;	# watches disabled, indexed by watch

my %watch;		# main configuration file data structure

my %alias;		# aliases

my %groups;		# hostgroups, indexed by group

my %views;		# view lists, indexed by name

my %view_users;         # view preferences, per user

#

# I/O routine globals

#

my %clients;		# fds of connected clients

my $numclients;		# count of connected clients

my %running;		# procs which are forked and running,

			# indexed by group/service

my $iovec;		# used for select loop

my %runningpid;		# procs which are forked and running,

			# indexed by PID

my $procs;		# number of outstanding procs

my %fhandles;		# input file handles of children

my %ibufs;		# buffer structure to hold data from children

my ($fdset_rbits, $fdset_ebits);

#

# history globals

#

my @last_alerts;	# alert history, in memory

my @last_failures;	# failure history, in memory

#

# misc. globals

#

my $i;			# loop iteration counter, used for debugging only

my $lasttm;		# the last time(2) the mon loop started

my $pid_file_owner;	# set when creating pid file

my $tm;			# used in main loop

#

# authentication structure globals

#

my %AUTHCMDS;

my %NOAUTHCMDS;

my %AUTHTRAPS;

#

# PAM authentication globals (must not be lexically scoped)

#

use vars qw ( $PAM_username $PAM_password ) ;

#

# opstatus globals

#

my (%OPSTAT, %FAILURE, %SUCCESS, %WARNING);	# operational statuses

my ($TRAP_COLDSTART, $TRAP_WARMSTART,		# trap types

	$TRAP_LINKDOWN, $TRAP_LINKUP,

	$TRAP_AUTHFAIL, $TRAP_EGPNEIGHBORLOSS,

	$TRAP_ENTERPRISE, $TRAP_HEARTBEAT);

my ($STAT_FAIL, $STAT_OK, $STAT_COLDSTART,	# _op_status values

	$STAT_WARMSTART, $STAT_LINKDOWN,

	$STAT_UNKNOWN, $STAT_TIMEOUT,

	$STAT_UNTESTED, $STAT_DEPEND, $STAT_WARN);

my ($FL_MONITOR, $FL_UPALERT,			# alert type flags

	$FL_TRAP, $FL_TRAPTIMEOUT,

	$FL_STARTUPALERT, $FL_TEST, $FL_REDISTRIBUTE,

        $FL_ACKALERT, $FL_DISABLEALERT);

my $TRAP_PDU;

my (%ALERTHASH, %MONITORHASH);			# hash of pathnames for

						# alerts/monitors

my $PROT_VERSION;

my $START_TIME;					# time(2) server started

my $TRAP_PRO_VERSION;				# trap protocol version

my $DEP_EVAL_SANDBOX;				# perl environment for

						# dep evals

#

# argument parsing

#

my $getopt_result = GetOptions(\%opt,

			       qw/

				  A|authfile=s

				  B|cfbasedir=s

				  D|statedir=s

				  L|logdir=s

				  M|m4:s

				  O|syslogfacility=s

				  P|pidfile=s

				  S|stopped

				  a|alertdir=s

				  b|basedir=s

				  c|configfile=s

				  d|debug+

				  f|fork

				  h|help

				  i|sleep=i

				  k|maxkeep=i

				  l|loadstate:s

				  m|maxprocs=i

				  p|port=i

				  r|randstart=s

				  s|scriptdir=s

				  t|trapport=i

				  v|version

				  /);

if (!$getopt_result) {

  usage();

  exit;

}

#

# these two things can be taken care of without

# initializing things further

#

if ($opt{"v"}) {

    print "$RCSID\n$RELEASE\n";

    exit;

}

if ($opt{"h"}) {

    usage();

    exit;

}

if ($opt{"d"})

{

    eval 'require Data::Dumper;';

    if ($@ ne "")

    {

    	die "error: $@\n";

    }

}

if ($^O eq "linux" || $^O =~ /^(open|free|net)bsd$/ || $^O eq "aix")

{

    Sys::Syslog::setlogsock ('unix');

}

elsif ($^O eq "solaris")

{

    Sys::Syslog::setlogsock ('stream');

}

openlog ("mon", "cons,pid", $CF{"SYSLOG_FACILITY"});

#

# definitions

#

die "basedir $opt{b} does not exist\n" if ($opt{"b"} && ! -d $opt{"b"});

init_globals();

init_cf_globals();

syslog_die ("config file $CF{CF} does not exist") if (! -f $CF{"CF"});

#

# read config file

#

if ((my $err = read_cf ($CF{"CF"}, 1)) ne "") {

    syslog_die ("$err");

}

closelog;

openlog ("mon", "cons,pid", $CF{"SYSLOG_FACILITY"});

#

# cmdline args override config file

#

$CF{"ALERTDIR"}  = $opt{"a"} if ($opt{"a"});

$CF{"BASEDIR"}   = $opt{"b"} if ($opt{"b"});

$CF{"AUTHFILE"}  = $opt{"A"} if ($opt{"A"});

$CF{"LOGDIR"}    = $opt{"L"} if ($opt{"L"});

$CF{"STATEDIR"}  = $opt{"D"} if ($opt{"D"});

$CF{"SCRIPTDIR"} = $opt{"s"} if ($opt{"s"});

$CF{"PIDFILE"}   = $opt{"P"} if defined($opt{"P"});	# allow empty pidfile

$CF{"MAX_KEEP"}  = $opt{"k"} if ($opt{"k"});

$CF{"MAXPROCS"}  = $opt{"m"} if ($opt{"m"});

$CF{"SERVPORT"}  = $opt{"p"} if ($opt{"p"});

$CF{"TRAPPORT"}  = $opt{"t"} if ($opt{"t"});

$SLEEPINT  = $opt{"i"} if ($opt{"i"});

if ($opt{"r"}) {

    syslog_die ("bad randstart value") if (!defined (dhmstos ($opt{"r"})));

    $CF{"RANDSTART"} = dhmstos($opt{"r"});

}

if ($opt{"S"}) {

    $STOPPED = 1;

    $STOPPED_TIME = time;

}

#

# do some path cleanups and

# build lookup tables for alerts and monitors

#

normalize_paths();

gen_scriptdir_hash();

if ($opt{"d"}) {

    debug_dir();

}

#

# load the auth control, bind, and listen

#

load_auth (1);

load_view_users(1);

#

# init client interface

#   %clients is an I/O structure, indexed by the fd of the client

#   $numclients is the number of clients currently connected

#   $iovec is fd_set for clients and traps

#

%clients = ();

$numclients = 0;

$iovec = '';

setup_server();

#

# fork and become a daemon

#

init_dtlog() if ($CF{"DTLOGGING"});

daemon() if ($opt{"f"});

if ($CF{"PIDFILE"} ne '' && open PID, ">$CF{PIDFILE}") {

    $pid_file_owner = $$;

    print PID "$pid_file_owner\n";

    close PID;

}

set_last_test ();

#

# randomize startup checks if asked to

#

randomize_startdelay() if ($CF{"RANDSTART"});

@last_alerts = ();

@last_failures = ();

readhistoricfile ();

$procs = 0;

$i=0;

$lasttm=time;

$fdset_rbits = $fdset_ebits = '';

%watch_disabled = ();

$SIG{HUP} = \&reset_server;

$SIG{INT} = \&handle_sigterm;		# for interactive debugging

$SIG{TERM} = \&handle_sigterm;

$SIG{PIPE} = 'IGNORE';

#

# load previously saved state

#

if (exists $opt{"l"}) {

    if ($opt{"l"}) {

	# If -l was given an argument (all, disabled, opstatus, etc...)

	# pass that to load_state

	load_state($opt{"l"});

    }else{

	# Otherwise default to old behavior of just loading disabled hosts/services/groups

	load_state("disabled");

    }

}

syslog ('info', "mon server started");

#

# startup alerts

#

do_startup_alerts();

#

# main monitoring loop

#

for (;;) {

debug (1, "$i" . ($STOPPED ? " (stopped)" : "") . "\n");

    $i++;

    $tm = time;

    #

    # step through the watch groups, decrementing and

    # handing expired timers

    #

    if (!$STOPPED) {

	if (defined $CF{"EXCLUDE_PERIOD"}

	    && $CF{"EXCLUDE_PERIOD"} ne "" &&

	    inPeriod (time, $CF{"EXCLUDE_PERIOD"})) {

	    debug (1, "not running monitors because of global exclude_period\n");

	} else {

	    foreach my $group (keys %watch) {

		foreach my $service (keys %{$watch{$group}}) {

		    my $sref = \%{$watch{$group}->{$service}};

		    my $t = $tm - $lasttm;

		    $t = 1 if ($t <= 0);

		    #

		    # trap timer

		    #

		    if ($sref->{"traptimeout"}) {

			$sref->{"_trap_timer"} -= $t;

			if ($sref->{"_trap_timer"} <= 0 && 

			    $tm - $sref->{"_last_trap"} > $sref->{"traptimeout"}) 

			  {

			      $sref->{"_trap_timer"} = $sref->{"traptimeout"};

			      handle_trap_timeout ($group, $service);

			  }

		    }

		    #

		    # trap duration timer

		    #

		    if (defined ($sref->{"_trap_duration_timer"})) {

			$sref->{"_trap_duration_timer"} -= $t;

			if ($sref->{"_trap_duration_timer"} <= 0) {

			    set_op_status ($group, $service, $STAT_OK);

			    undef $sref->{"_trap_duration_timer"};

			}

		    }

		    #

		    # polling monitor timer

		    #

		    if ($sref->{"interval"} && $sref->{"_timer"} <= 0 &&

			!$running{"$group/$service"})

		      {

			  if (!$CF{"MAXPROCS"} || $procs < $CF{"MAXPROCS"})

			    {

				if (defined $sref->{"exclude_period"} 

				    && $sref->{"exclude_period"} ne "" &&

				    inPeriod (time, $sref->{"exclude_period"}))

				  {

				      debug (1, "not running $group,$service because of exclude_period\n");

				  }

				elsif (($sref->{"dep_behavior"} eq "m" &&

					defined $sref->{"depend"} && $sref->{"depend"} ne "")

				       || (defined $sref->{"monitordepend"} && $sref->{"monitordepend"} ne "")) 

				  {

				      if (dep_ok ($sref, 'm'))

					{

					    run_monitor ($group, $service);

					}

				      else

					{

					    debug (1, "not running $group,$service because of depend\n");

					}

				  }

				else

				  {

				      run_monitor ($group, $service);

				  }

			    }

			  else

			    {

				syslog ('info', "throttled at $procs processes");

			    }

		      }

		    else

		      {

			  $sref->{"_timer"} -= $t;

			  if ($sref->{"_timer"} < 0)

			    {

				$sref->{"_timer"} = 0;

			    }

		      }

		}

	    }

	}

    }

    $lasttm = time;

    #

    # collect any output from subprocs

    #

    collect_output;

    #

    # clean up after exited processes, and trigger alerts

    #

    proc_cleanup;

    #

    # handle client, server, and trap I/O

    # this routine sleeps for $SLEEPINT if no I/O is ready

    #

    handle_io;

}

die "not reached";

END {

    unlink $CF{"PIDFILE"} if $$ == $pid_file_owner && $CF{"PIDFILE"} ne '';

}

##############################################################################

#

# startup alerts

#

sub do_startup_alerts {

    foreach my $group (keys %watch) {

    	foreach my $service (keys %{$watch{$group}}) {

	    do_alert ($group, $service, "", 0, $FL_STARTUPALERT);

	}

    }

}

#

# handle alert event, throttling the alert call if necessary

#

sub do_alert {

    my ($group, $service, $output, $retval, $flags) = @_;

    my (@groupargs, $last_alert, $alert);

    my ($sref, $range, @alerts);

debug (1, "do_alert flags=$flags\n");

    $sref = \%{$watch{$group}->{$service}};

    my $tmnow = time;

    if ($STOPPED) {

      syslog ("notice", "ignoring alert for $group,$service because the mon scheduler is stopped");

      return;

    }

    #

    # if redistribute it set, call it now

    #

    if ($sref->{"redistribute"} ne '') 

    {

        my ($fac, $args);

        ($fac, $args) = split (/\s+/, $sref->{"redistribute"}, 2);

        call_alert (

                    group       => $group,

                    service     => $service,

                    output      => $output,

                    retval      => $retval,

                    flags       => $flags | $FL_REDISTRIBUTE,

                    alert       => $fac,

                    args        => $args,

                   )

    }

    #

    # if the alarm is disabled, ignore it

    #

    if ((exists $watch_disabled{$group} && $watch_disabled{$group} == 1) 

	|| (defined $sref->{"disable"} && $sref->{"disable"} == 1))

    {

	syslog ("notice", "ignoring alert for $group,$service");

	return;

    }

    #

    # dependency check

    #

    if (!($flags & $FL_STARTUPALERT) &&

	!($flags & $FL_UPALERT) &&

	((defined $sref->{"depend"} && $sref->{"dep_behavior"} eq "a")

	 || (defined $sref->{"alertdepend"})))

    {

	if (!$sref->{"_depend_status"})

	{

	    debug (1, "alert for $group,$service supressed because of dep fail\n");

	    return;

	}

    }

    my ($summary) = split("\n", $output);

    $summary = "(NO SUMMARY)" if (!defined $summary || $summary =~ /^\s*$/m);

    my ($prevsumm) = split("\n", $sref->{"_failure_output"}) if (defined $sref->{"_failure_output"});

    $prevsumm = "(NO SUMMARY)" if (!defined $prevsumm || $prevsumm =~ /^\s*$/m);

    my $strippedsummary = $summary;

    $strippedsummary =~ s/\s//mg;

    my $strippedprevious = $prevsumm;

    $strippedprevious =~ s/\s//mg;

    # If the summary changed, un-acknowledge the service if 'unack_summary' is set

    if ($sref->{'_ack'} != 0 

	&& $sref->{'unack_summary'} == 1 

	&& $strippedsummary ne $strippedprevious

	&& !($flags & ($FL_UPALERT|$FL_ACKALERT|$FL_DISABLEALERT))) {

	print STDERR "Unacking $group/$service:\nSummary: X".$strippedsummary."X\nPrevious: X".$strippedprevious."X\n";

	$sref->{"_ack"} = 0;

	$sref->{"_ack_comment"} = "";

        $sref->{"_consec_failures"}=1;

        foreach my $period (keys %{$sref->{"periods"}})

          {

            $sref->{"periods"}->{$period}->{"_last_alert"} = 0;

#            $sref->{"periods"}->{$period}->{"_alert_sent"} = 0;

            $sref->{"periods"}->{$period}->{"_1stfailtime"} = 0;

            $sref->{"periods"}->{$period}->{"_failcount"} = 0;

          }

    }

    #

    # no alerts for ack'd failures, except for upalerts or summary changes

    # when observe_summary is set

    #

    if ($sref->{"_ack"} != 0 && !($flags & ($FL_UPALERT|$FL_ACKALERT|$FL_DISABLEALERT)))

    {

	syslog ("debug", "no alert for $group.$service" .

		" because of ack'd failure");

	return;

    }

    #

    # check each time period for pending alerts

    #

    foreach my $periodlabel (keys %{$sref->{"periods"}})

    {

	#

	# only send alerts that are in the proper period

	#

    	next if (!inPeriod ($tmnow, $sref->{"periods"}->{$periodlabel}->{"period"}));

    	my $pref = \%{$sref->{"periods"}->{$periodlabel}};

	#

	# skip upalerts/ackalerts not paired with down alerts

	# disable by setting "no_comp_alerts" in period section

	#

	if (!$pref->{"no_comp_alerts"} && ($flags & ($FL_UPALERT | $FL_ACKALERT)) && !$pref->{"_alert_sent"})

	{

	    syslog ('debug', "$group/$service/$periodlabel: Suppressing upalert since no down alert was sent.") if ($flags & $FL_UPALERT);

	    syslog ('debug', "$group/$service/$periodlabel: Suppressing ackalert since no down alert was sent.") if ($flags & $FL_ACKALERT);

	    next;

	}

        #

        # skip looping upalerts when "no_comp-alerts" set.

        #

        if ($pref->{"no_comp_alerts"} && ($flags & $FL_UPALERT) && ($pref->{"_no_comp_alerts_upalert_sent"}>0))

        {   

            next;

        }

	#

	# do this if we're not handling an upalert, startupalert, ackalert, or disablealert

	#

	if (!($flags & $FL_UPALERT) && !($flags & $FL_STARTUPALERT)  && !($flags & $FL_DISABLEALERT) && !($flags & $FL_ACKALERT))

	{

  	    #

	    # alert only when exit code matches

	    #

	    if (exists $pref->{"alertexitrange"}) {

		next if (!inRange($retval, $pref->{"alertexitrange"}));

	    }

	    #

	    # alert only numalerts

	    #

	    if ($pref->{"numalerts"} &&

	    	     $pref->{"_alert_sent"} >= $pref->{"numalerts"})

	    {

                syslog ('debug', "$group/$service/$periodlabel: Suppressing alert since numalerts is met.");

	    	next;

	    }

	    #

	    # only alert once every "alertevery" seconds, unless

	    # output from monitor is different or if strict alertevery

	    #

	    # strict and _ignore_summary are basically the same though

	    # strict short-circuits and overrides other settings and exists

	    # for compatibility with pre-1.1 configs

	    #

	    if	($pref->{"alertevery"} != 0 &&                                                                 # if alertevery is set and

		 ($tmnow - $pref->{"_last_alert"} < $pref->{"alertevery"}) &&                                  # we're within the time period and one of these:

		 (($pref->{"_alertevery_strict"}) ||                                                           # [ strict is set or

		  ($pref->{"_observe_detail"} && $sref->{"_failure_output"} eq $output) ||                     # observing detail and output hasn't changed or

		  (!$pref->{"_observe_detail"} && (!$pref->{"_ignore_summary"}) && ($prevsumm eq $summary)) || # not observing detail

		    											       # and not ignoring summary and summ hasn't changed or

		  ($pref->{"_ignore_summary"})))	                                                       # we're ignoring summary changes ]

	    {

                syslog ('debug', "$group/$service/$periodlabel: Suppressing alert for now due to alertevery.");

		next;

	    }

	    #

	    # alertafter NUM

	    #

	    if (defined $pref->{"alertafter_consec"} && ($sref->{"_consec_failures"} < $pref->{"alertafter_consec"}))

	    {

                syslog ('debug', "$group/$service/$periodlabel: Suppressing alert for now due to alertafter consecutive failures.");

	    	next;

	    }

	    #

	    # alertafter timeval

	    #

	    elsif ( (!defined ($pref->{"alertafter"})) && (defined ($pref->{"alertafterival"})) )

	    {

	    	$pref->{'_1stfailtime'} = $tmnow if $pref->{'_1stfailtime'} == 0;

		if ($tmnow - $pref->{'_1stfailtime'} <= $pref->{'alertafterival'})

		{

                    syslog ('debug', "$group/$service/$periodlabel: Suppressing alert for now due to alertafter numval.");

		    next;

		}

	    }

	    #

	    # alertafter NUM timeval

	    #

	    elsif (defined ($pref->{"alertafter"}))

	    {

		$pref->{"_failcount"}++;

		if ($tmnow - $pref->{'_1stfailtime'} <= $pref->{'alertafterival'} &&

		    $pref->{"_failcount"} < $pref->{"alertafter"})

		{

                    syslog ('debug', "$group/$service/$periodlabel: Suppressing alert for now due to alertafter num timeval.");

		    next;

		}

		#

		# start a new time interval

		#

		if ($tmnow - $pref->{'_1stfailtime'} > $pref->{'alertafterival'})

		{

		    $pref->{"_failcount"} = 1;

		}

		if ($pref->{"_failcount"} == 1)

		{

		    $pref->{"_1stfailtime"} = $tmnow;

		}

		if ($pref->{"_failcount"} < $pref->{"alertafter"})

		{

                    syslog ('debug', "$group/$service/$periodlabel: Suppressing alert for now due to alertafter num timeval.");

		    next;

		}

	    }

	}

	#

	# at this point, no alerts are blocked,

	# so send the alerts

	#

	#

	# trigger multiple alerts in this period

	#

	if ($flags & $FL_UPALERT)

	{

	    @alerts = @{$pref->{"upalerts"}};

	}

	elsif ($flags & $FL_STARTUPALERT)

	{

	    @alerts = @{$pref->{"startupalerts"}};

	}

	elsif ($flags & $FL_DISABLEALERT)

	{

	    @alerts = @{$pref->{"disablealerts"}};

	}

	elsif ($flags & $FL_ACKALERT)

	{

	    @alerts = @{$pref->{"ackalerts"}};

	}

	else

	{

	    @alerts = @{$pref->{"alerts"}};

	}

	my $called = 0;

	for (my $i=0;$i<@alerts;$i++)

	{

	    my ($range, $fac, $args);

	    if ($alerts[$i] =~ /^exit\s*=\s*((\d+|\d+-\d+))\s/i)

	    {

		$range=$1;

		next if (!inRange($retval, $range));

		($fac, $args) = (split (/\s+/, $alerts[$i], 3))[1,2];

	    }

	    else

	    {

		($fac, $args) = split (/\s+/, $alerts[$i], 2);

	    }

	    $called++ if (call_alert (

		    group	=> $group,

		    service	=> $service,

		    output	=> $output,

		    retval	=> $retval,

		    flags	=> $flags,

		    pref	=> $pref,

		    alert	=> $fac,

		    args	=> $args,

		)

	    );

	}

	#

	# reset _alert_sent if up alert was sent from a trap

	#

        if ($called)

        {

            if( (($FL_TRAP | $flags) && ($FL_UPALERT & $flags)) ) {

	        $pref->{"_alert_sent"} = 0;

                $pref->{"_last_alert"} = 0;

            }

            else {

                $pref->{"_alert_sent"}++;

                #

                # reset _no_comp_alerts_upalert_sent counter - when service will be

                # back up, upalert will be sent.

                #

                if ($pref->{"no_comp_alerts"}) {

                    $pref->{"_no_comp_alerts_upalert_sent"} = 0;

                }

            }

	    if ($pref->{"no_comp_alerts"} && ($flags & $FL_UPALERT)) {

		$pref->{"_no_comp_alerts_upalert_sent"}++;

	    }

        }

    }

}

#

# walk through the watch list and reset the time

# the service was last called

#

sub set_last_test {

    my ($i, $k, $t);

    $t = time;

    foreach $k (keys %watch)

    {

    	foreach my $service (keys %{$watch{$k}})

	{

	    $watch{$k}->{$service}->{"_timer"} = $watch{$k}->{$service}->{"interval"};

	}

    }

}

#

# parse configuration file

#

# build the following data structures:

#

# %group

#       each element of %group is an array of hostnames

#       group records are terminated by a blank line in the

#       configuration file

# %watch{"group"}->{"service"}->{"variable"} = value

# %alias

#

sub read_cf {

    my ($CF, $commit) = @_;

    my ($var, $watchgroup, $ingroup, $curgroup, $inwatch,