|
Softpanorama |
May the source be with you, but remember the KISS principle ;-)
|
| News | See also | Recommended Links | Perl | ||
|
TFM Perl Scripts |
TEC Perl Scripts | Monitoring Perl Scripts | Humor | Etc |
|
#!/etc/Tivoli/bin/perl
# tivstat.pl
# Script to watch for critical tivoli processes dying.
#
# Notes:
# 1) Cache keys are forced to lowercase
# status values of cache hash:
# no key - this process has never been detected on this endpoint
# 1 - this process is running
# 0 - this process was running but has stopped
# 2) Includes work done by James Boone (see tme10 email forum "postemsg.pl")
# 3) Intended to be run as an lcfd boot_method with its own dependency set:
# on NT,
# - ntprocinfo.exe (generates a process list on NT - Tivoli supplied)
# - perl.exe (V4 Tivoli supplied)
# - tivstat.tecs (data file that has to be created)
# on UNIX,
# - perl (V4 Tivoli supplied)
# - tivstat.tecs (data file that has to be created)
#
# V1.0, Original, Paul Claridge, 06Jun2001
# V1.1 Paul Claridge, 12Jun2001
# Added more logic to correctly detect oserv restart
# Tightened up string matches to avoid bad process matching
# V1.2 Paul Claridge, 13Jun2001
# Further tightening of string matching to avoid wrong alerts in the
# case where a process is a substring of another process
# You probably need to review all of these and the %tiv_sev hash!
$ENV{'UNIX95'}=1; # improves portability for ps on UNIXs
$want_procs=1; # 1 - check processes, 0 - don't
$want_hb=1; # 1 - generate heartbeat, 0 - don't
$proc_class='Tivoli_process;source=ALL;'; # class and source for process events
$heartbeat_class='Tivoli_heartbeat;source=ALL;'; # class and source for hearbeat events
$proc_poll=$next_procs=20; # interval in seconds for looking at the process list
$hb=$next_hb=60; # interval in seconds for sending a heartbeat
$do_procs=$want_procs; # set flag for first time through
$do_hb=$want_hb; # set flag for first time through
%tiv_sev=('tmf_sched','FATAL', # associate a TEC severity with each process.
'ep_mgr','FATAL',
'rptm','FATAL',
'oserv','FATAL',
'lcfd','MINOR',
'gateway','FATAL',
'rim_oracle_prog','FATAL',
'trip','MINOR',
'spider','MINOR',
'rcserv','MINOR',
'ntfserver','CRITICAL',
'sentry_engine','CRITICAL',
'sentry_gateway','CRITICAL',
'tec_server','CRITICAL',
'tec_gateway','CRITICAL',
'oracle73','CRITICAL',
'tnslsnr','CRITICAL',
'tecadnts','CRITICAL',
'tecad_snmps','CRITICAL',
'dm_ep_engine','CRITICAL');
# make list of tivoli procs of interest
@tivprocs=keys %tiv_sev;
# associate command to generate proc list with interp
%proc_gen=('w32-ix86','ntprocinfo','aix4-r1','ps -eo pid,args','solaris2','ps -eo pid,args');
$proc_cmd=$proc_gen{$ENV{'INTERP'}};
# determine files location
$dir=$ENV{'LCF_DATDIR'} || $ENV{'PWD'} || '.';
$cache_file="$dir/tivstat.cache";
$log="$dir/tivstat.log";
$tec_list="$dir/tivstat.tecs";
$pid="$dir/tivstat.pid";
&init; # check that we are not running already, open files etc
# determine endpoint name
chop($h=`hostname`);
$me= &get_ep_name || $h || $ENV{'COMPUTERNAME'} || $ENV{'HOST'};
# set up signal handlers (NT only seems to catch Cntrl-C)
for $s (HUP,INT,QUIT,KILL) { $SIG{$s}='signals'; }
# setup some static event information
$origin=join('.',unpack(C4,(gethostbyname($me))[4]));
$static="hostname=$me;origin=$origin;";
# loop continuously with variable sleeps
while (1) {
$loop_start=time; # capture loop start
$pmsg=''; # reset packed message of event(s)
&check_procs if $do_procs && $want_procs;
&create_heartbeat if $do_hb && $want_hb;
if ($pmsg) { # send any events in one go, try tecs in order
for $t (@tec_servers) {
($techost,$pm_flag,$tec_recv)=split(/:/,$t);
last if &send_events($tec_recv);
next unless $pm_flag;
# if this tec fails and is portmapped retry once in case of restart
unless ($retried) {
$new_triple=join(':',$techost,$pm_flag,pack('S n a4 x8',2,&get_port($techost),$tec_addrs{$techost}));
$t=~s/^(.*)$/$new_triple/;
$retried=1; redo; # try that one again
}
else { $retried=0; }
}
}
print CACHE join(',',%cache); seek(CACHE,0,0); # just keep latest values (1 line)
if ($t=&calc_wait(time)) { sleep $t; }
}
#### subroutines ####
sub init {
# open log file for errors etc
open(LOG,">>$log") || warn "Could not open log file <$log>\n";
select LOG; $|=1;
unless ($want_procs || $want_hb)
{ print LOG &now." # Neither \$want_procs nor \$want_hb flags have been set, exiting..\n" if LOG; exit; }
# check we have command for this interp
if ($proc_cmd) { &get_procs; }
else { print LOG &now." # Do not know process command to run for interp <$ENV{'INTERP'}>\n" if LOG; exit; }
# check for already running tivstat and exit if found
if (open(PID,"$pid")) {
$last_pid=<PID>;
exit 0 if grep(/"$last_pid:perl"/,@p);
close PID;
}
open(PID,">$pid"); print PID $$; close PID; # write new pid file
print LOG &now." # --- started, pid <$$>\n" if LOG;
# read disk file if it exists - saved cache across restarts/reboots
if (open(IN,"$cache_file")) { %cache=split(',',<IN>); close IN; } else { %cache=(); }
# open new cache file for writing latest cache
if (open(CACHE,">$cache_file")) { select CACHE; $|=1; }
else { print LOG &now." # Could not open cache file <$cache_file>\n" if LOG; }
# read tivstat.tecs file for tecserver(s)
if (open(TL,"$tec_list")) {
chop($tecs=<TL>); close TL;
@tec_servers=split(',',$tecs);
for $t (@tec_servers) {
local($techost,$portmapper_flag)=split(/:/,$t);
$tec_addrs{$techost}=(gethostbyname($techost))[4];
$portmapper_flag ? ($port=&get_port($techost)) : ($port=5529);
$t.=":".pack('S n a4 x8',2,$port,$tec_addrs{$techost});
}
}
else { @tec_servers=(); print LOG &now." # No TEC servers defined in <$tec_list>, unable to send events!\n" if LOG; }
}
sub get_procs {
(@running,@not_running)=(); # reset for each run
@p=grep(s/^\s*(\d*)\s*(\S*).*$/"$1:$2"/,`$proc_cmd`); # make list of running processes
for $p (@tivprocs) { (grep(/\b$p\b/i,@p)) ? push(@running,"\L$p") : push(@not_running,"\L$p"); }
}
sub check_procs {
&get_procs;
# catch oserv restart
if (grep(/^oserv$/,@running) && defined $cache{'oserv'} && $cache{'oserv'}==0) {
&create_proc_event('oserv','restarted',HARMLESS);
$cache{'oserv'}++;
# now check children have restarted
while ($r=shift @o_children) {
if (grep(/^$r$/,@running)) { $cache{$r}++; }
else { &create_proc_event($r,'not restarted with oserv',$tiv_sev{$r}); }
}
}
for $r (@running) {
next if $cache{$r};
&create_proc_event($r,'restarted',HARMLESS) if grep(/^$r$/,keys %cache);
$cache{$r}++;
}
# catch oserv failure to send only one event
if (grep(/^oserv$/,@not_running) && $cache{'oserv'}) { $o_stop=1; } else { $o_stop=0; }
for $n (@not_running) {
$cache{$n} ? $cache{$n}-- : next;
if ($n=~/^oserv$/ || !$o_stop) { &create_proc_event($n,'stopped',$tiv_sev{$n}); }
if ($n!~/^oserv$/ && $o_stop) { push(@o_children,$n); }
}
}
sub calc_wait {
$next_procs-=($_[0]-$loop_start);
$next_hb-=($_[0]-$loop_start);
if ($next_procs <= 0 || $next_hb <= 0 ) {
if ($next_procs <= 0) { $do_procs=1; $next_procs+=$proc_poll; } else { $do_procs=0; }
if ($next_hb <= 0) { $do_hb=1; $next_hb+=$hb; } else { $do_hb=0; }
return 0;
}
elsif ($next_procs > $next_hb) { $wait=$next_hb; $do_hb=1; $do_procs=0; $next_procs-=$next_hb; $next_hb=$hb; }
elsif ($next_procs < $next_hb) { $wait=$next_procs; $do_procs=1; $do_hb=0; $next_hb-=$next_procs; $next_procs=$proc_poll; }
else { $wait=$next_procs; $do_procs=1; $do_hb=1; $next_procs=$proc_poll; $next_hb=$hb; }
return $wait;
}
sub create_heartbeat {
local($date)="date=".&now;
local($msg)=$heartbeat_class.$static."$date;severity=HARMLESS;".'END';
$msg_l=length($msg);
$term_l=$msg_l+2; # text terminated by newline and Cntrl-A (0x01)
local($packed)=pack("a8 N7 a$msg_l c2","<START>>",0,0,0,0,0,$term_l,$term_l,$msg,0x0a,0x01);
$pmsg.=$packed;
}
sub create_proc_event {
local($proc,$mode,$sev)=@_;
local($date)="date=".&now;
local($msg)=$proc_class.$static."$date;msg=\"<$proc> has $mode.\";severity=\U$sev;".'END';
print LOG &now." # <$msg>\n" if LOG;
$msg_l=length($msg);
$term_l=$msg_l+2; # text terminated by newline and Cntrl-A (0x01)
local($packed)=pack("a8 N7 a$msg_l c2","<START>>",0,0,0,0,0,$term_l,$term_l,$msg,0x0a,0x01);
$pmsg.=$packed;
}
sub now {
local(@t)=localtime; $t[4]++; $t[5]+=1900;
for (@t){ $_=~s/(.*)/sprintf("%02d",$_)/e; }
return "$t[3]/$t[4]/$t[5]-$t[2]:$t[1]";
}
sub send_events {
if (socket(S,2,1,6) && connect(S,$_[0])) {
select S; $|=1; $rc=send(S,$pmsg,0); close S;
unless ($rc) { print LOG &now." # ERROR sending event(s) to <$techost>\n" if LOG; }
}
else { print LOG &now." # socket connect error to <$techost>\n" if LOG && (!$pm_flag || $retried); $rc=0; }
return $rc;
}
# send udp packet to portmapper to get tec recv port
sub get_port {
local($dg)=pack('N13 x2',12345,0,2,0x0186a0,2,3,0,0,0,0,100033057,1,6);
local($tec_pm)=pack('S n a4 x8',2,111,$tec_addrs{$_[0]});
socket(PM,2,2,17) || return -1;
send(PM,$dg,0,$tec_pm) || return -1;
recv(PM,$reply,256,0);
close PM;
return (unpack(N7,$reply))[6];
}
sub get_ep_name {
if (open(CFG,"$ENV{'LCF_DATDIR'}/last.cfg")) {
local($ep_name)= grep(do{chop;s/^lcs.machine_name=(.*)$/$1/},<CFG>);
close CFG;
return $ep_name;
}
}
sub signals { print LOG &now." # Caught signal <$_[0]>, exiting..\n" if LOG; close CACHE; close LOG; exit; }
- General design: A daemon regulary polls the TEC server for new events. For each of the defined event groups the most recent event of each severity is read. This data is stored in files, one for each event group. A client program regulary reads one of these files. Which one depends on the event group the user is interested in. It checks each severity for a changed date since last access. A sound for the highest changed severity is being played. The Tec-o-Bell parts are very lightweight and do not slow down framework functionality. Event storms do not result in additional workload. Only short polling cycles could result in some reasonable CPU usage. The number of clients is nearly unlimited. Due to the polling mechanism of server daemon and client the alarm sounds are being played with some time delay. The primary goal was to serve many clients with little resource usage. So hearing an alarm one minute late seemed to be even better than not seeing a severe error because another desktop window hides the TEC console. - How to get it: This program is FREE (GPL) for anyone! To get a copy mailto:robert.widmer@omv.com copyright (c) 2000 by Robert Widmer robert.widmer@omv.com
Do you want to see TEC events in a web browser? and group this events in a business view tree? and have alarm sounds whenever a new event arrives? This is now available with "WebTEC". It's a tool I developed for our Tivoli environment. And it is now usable FREE (GPL) for anyone! Have a look at http://members.mcnon.com/widi/webtec
Key features:
1. multiple business view trees to a set of event groups
2. http based: on the client side only a standard web browser is needed nearly unlimited number of clients no firewall problems, access even through internet Framework functionality is only used to acknowledge/close events server part is based on Apache and PHP3, two great freeware products
3. event sensitive help documents with index engine
4. fast and easy configuration, event filters based on SQL syntax
5. configurable alarm sounds
6. asynchronous event display, unaffected by event storms
7. free software
Much more details and a limited "look and feel" prototype is available at
http://members.mcnon.com/widi/webtec
Robert Widmer robert.widmer@omv.com
NAME
Tivoli - Perl extension for Tivoli TME10SYNOPSIS
Not (yet) Autoloader implemented.use Tivoli::DateTime; use Tivoli::Logging; use Tivoli::Fwk; use Tivoli::Endpoints;VERSION
v0.01License
Copyright (c) 2001 Robert Hase. All rights reserved. This program is free software; you can redistribute it and/or modify it under the same terms as Perl itself.DESCRIPTION
This Module will handle about everything you may need for Tivoli TME10. If anything has been left out, please contact me at tivoli.rhase@muc-net.de so it can be added.DETAILS
This Module is still in work.The following Packages are included in v0.01 (read Package-Documentations for Details)
This Package will handle about everything you may need for handling the date / time. This Package will handle about everything you may need for Logging. This Package will handle about everything you may need for Framework. This Package will handle about everything you may need for Endpoints.DEPENDENCIES
The Tivoli Environment should be sourced at firstPlattforms and Requirements
read Package-Documentations for Details
tested on:- w32-ix86 (Win9x, NT4, Windows 2000) - aix4-r1 (AIX 4.3) - Linux (Kernel 2.2.x)
requires Perl v5 or higher
HISTORY
MODULE AUTHOR VERSION DATE ---------------------------------------------------- Tivoli RHase 0.01 2001-08-05PACKAGE AUTHOR VERSION DATE ---------------------------------------------------- DateTime RHase 0.03 1999 Logging RHase 0.02 2001-07-18 Fwk RHase 0.04 2001-07-06 Endpoints RHase 0.02 2001-07-06
INSTALLATION
UNIX
To install this module type the following:Win32
Still in workAUTHOR
Robert Hase ID : RHASE eMail : Tivoli.RHase@Muc-Net.de Web : http://www.Muc-Net.deSEE ALSO
CPAN http://www.perl.comTivoli http://www.tivoli.com
This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. Tivoli is a trademark of IBM corp.backpan.perl.org/authors/id/C/ CH/CHLIGE/Tivoli-AccessManager-Admin-0.1.readme - 2k -
#!/usr/bin/perl
#
# PSW Technologies, Inc. May 1999
#
# This tells who is running tec_client and matches process_id
#
open (ODSTAT,"odstat -c -v|");
while (
if (/([0-9\.]+)\#TecClient::tec_client\#.*/) {
$tec_client = $1;
$administrator = `idlcall $tec_client _get_label`;
$_ =
$pid=$1;
print "\n$administrator is running a TEC client PID=$pid";
}
}
print "\n\n";
http://members.mcnon.com/widi/webtec Do you want to see TEC events in a web browser? and group this events in a business view tree? and have alarm sounds whenever a new event arrives? This is now available with "WebTEC". It's a tool I developed for our Tivoli environment. And it is now usable FREE (GPL) for anyone! Have a look at
Key features:
Much more details and a limited "look and feel" prototype is available at http://members.mcnon.com/widi/webtec Robert Widmer robert.widmer@omv.com
Copyright © 1996-2008 by Dr. Nikolai Bezroukov. www.softpanorama.org was created as a service to the UN Sustainable Development Networking Programme (SDNP) in the author free time. Submit comments This document is an industrial compilation designed and created exclusively for educational use and is placed under the copyright of the Open Content License(OPL). Original materials copyright belong to respective owners. Quotes are made for educational purposes only in compliance with the fair use doctrine.
Standard disclaimer: The statements, views and opinions presented on this web page are those of the author and are not endorsed by, nor do they necessarily reflect, the opinions of the author present and former employers, SDNP or any other organization the author may be associated with. We do not warrant the correctness of the information provided or its fitness for any purpose.
Last modified: June 02, 2008