#!/opt/SUNWstade/bin/perl -I/opt/SUNWstade/lib
#<copyright>
# ----------------------------------------------------------
# Sun Proprietary/Confidential Code
# Copyright 2001, Sun Microsystems, Inc. All rights reserved.
# ----------------------------------------------------------
#</copyright>

#
#  $Id: rasagent,v 1.144 2004/08/17 21:59:36 mckenney Exp $
#
use strict;
use Getopt::Std;
use System;
use Solution;
use Health;
use Timelapse;
use Timer;
use PDM::ConfigFile;
use Debug;
use Repeat;
use UNIVERSAL qw( isa ) ;
use CIM;
use Discover;
use Rasagent;
use Robin;
use Snapshot;
use Process;

use vars qw(
   %CAT
   $util $MASTER_LOC $RASPORT  %opts  $LB $VERSION $SIGTERM   $LAST_INSP
   $HOME $DATA %DONE $CONF $CRON_LOC $SLAVE $PIDfile $renv $devices  
   $hosts $frus $device $notifs $Config $FORCE $HOST_ID $HOST_NAME %HMAP  %AMAP $SE_CFG
   $SINFO_DB $SINFO $MAX_DEVICES $DO_ALL_DEVICES  $TESTMODE
);

if ($ENV{OPTS}) {
  @ARGV = split(/ +/, $ENV{OPTS});
}

if (!getopts("PCb:raLIl:MHAfeD:hc?Pd:T:v", \%opts)) {
    print " $Getopt::Std::ERROR \n";
    usage();
    exit(1);
}

$TESTMODE = 1 if (exists $opts{T});
Debug->level($opts{d});

my($local_p) = $INC[0];
my($ix) = rindex($local_p, "/");

$HOME   = $opts{D} || substr($local_p,0,$ix);

System->set_home($HOME);

if ($opts{T}) { # only runs whne it's not -T0
   Snapshot->setup($opts{T}, $opts{b});
}
if ($opts{C}) {
   unlink "$HOME/DATA/CACHE.db";
   unlink "$HOME/DATA/MAXEVENTS.db";
}

$DO_ALL_DEVICES = $opts{a};        # keep going after finding a problem.
$opts{d}     = 2 if ($TESTMODE && !$opts{d});
my $AUDIT    = 1 if ($opts{A});
$FORCE       = 1 if ($opts{f});    # ignore diags.
my $key_list = $opts{l};

my $inspectors_on = $opts{I};

System->set_eventMaxOff($opts{M});
System->set_audit($AUDIT);
System->set_logDebug($opts{L});
System->set_allDevices($opts{r});  # ignore round robin

my $force_refresh;

$ENV{LANG}   = 'C';
$ENV{LC_ALL} = 'C';

System->set_testMode($TESTMODE || 0);
my($e) = $opts{e} || "2";
System->set_eventDir($e);

my($op);
$op .= "Test on,"    if ($TESTMODE);
$op .= "Debug on,"   if ($opts{d});
if ($op) {
  chop($op);
  print STDERR "Options: $op\n";
}

&version if ($opts{v});
if ($opts{"?"} || $opts{"h"}) {
  &usage;
  exit(0);
}

unlink "$HOME/DATA/snapshot.log";
my $RASAGENT = "$HOME/bin/rasagent";
$RASAGENT .= " -d $opts{d}" if ($opts{d});
$RASAGENT .= " -L" if ($opts{L});

if ($opts{c}) {  # from crons
  $CRON_LOC = "$HOME/System/cron_started";
  if (!-f $CRON_LOC) {
     open(O, ">" . $CRON_LOC); print O Util->get_today; close(O);
     Process->done("rasagent");
     exit;
  }
}

$LB = Labels->read('rasagent');









$VERSION   = "1.0";
$SIGTERM   = 15;
$DATA      = "$HOME/DATA";
$PIDfile   = "$DATA/pid";
$CONF      = "$DATA/rasagent.conf";

System->set_config($CONF);
System->set_schemaVersion("1.0");
System->set_runId(time);

$HOST_ID     = System->hostid();
$HOST_NAME   = System->hostname();
my $today    = Util->today();
System->set_ipno(Util->name2ip($HOST_NAME));

if (Debug->level() > 2) {
   print $LB->expand('new_snapshot', "/DATA/snapshot.log") . "\n";
}

$RASPORT     = System->getConfigPort($HOME);
System->set_rasport($RASPORT);
System->set_secure((-f "$HOME/DATA/SECURE") ? "Y" : "N");

PDM::ConfigFile->configFromMaster();
($renv, $devices, $hosts, $notifs, $Config) = PDM::ConfigFile->read();

if ($renv->{active} eq "N" && Debug->level()  < 2) {
  Debug->print2("Agent not active, aborting!");
  exit(1);
}

System->set_Config($Config);

Snapshot->fixRenv($renv) if ($opts{T});
System->set_renv($renv);

# CHECK Cluster SETUP
#
if ($renv->{active_cluster}) { # run only on the primary
  require Agent::CLUSTER;
  if (!Agent::CLUSTER->isPrimary()) {
    Debug->print2("CLUSTER: this host is not the primary node, aborting!");
    Process->done("rasagent");
    exit;
  }
}

Rasagent->check_cron($notifs, $PIDfile);


###############################################
#  READ CONFIGFILE (FROM MASTER)
###############################################
Util->clearLastCommand();

Rasagent->create_pid_file($LB, $PIDfile);

exit if (Rasagent->read_pid_file($LB, $PIDfile) != $$);

System->set_configDevices($devices);
System->set_configHosts($hosts);

$renv->{"timeout.luxadm"}  = 200 if (!$renv->{"timeout.luxadm"});
$renv->{"timeout.rm6"}     = 200 if (!$renv->{"timeout.rm6"});
$renv->{"timeout.discman"} = 600 if (!$renv->{"timeout.discman"});

if ($renv->{solution} ne "N") {
   System->set_allDevices(1);      # ignore round robin or racks
}

$MASTER_LOC          = Util->findMaster();
$renv->{hostid}      = $HOST_ID;

#$renv->{logfile}    = "/var/adm/messages" if (!$renv->{logfile});

my($s1) = ($MASTER_LOC) ? "SLAVE, MASTER=>$MASTER_LOC" : "MASTER";

$util = Util->new({  data  => $DATA, home  => $HOME, renv  => $renv });
  

###############################################
#  LOOK FOR DEVICES TO MONITOR
###############################################

require PDM;
require FSA;
require Modules;
require Scheduler;
require TO;
require Util::Http;
require Events;
require Agent;


my $pdm  = PDM->new({  dir => "$HOME/DATA",
                      renv => $renv ,
                   devices => $devices,
                     hosts => $hosts,
                    notifs => $notifs,
                    });

$MAX_DEVICES = 100;

$SINFO_DB       = RasDB->new("SEQUENCER");
$SINFO          = $SINFO_DB->hash();
System->set_SINFO($SINFO);
System->set_SINFODB($SINFO_DB);
my $scheduled_run;
my $static_list = [];

if ($key_list) {                     # DEVICE GIVEN AS ARGUMENT
   my @l = split(/\:/, $key_list);
   foreach my $k (@l) {
      my $dev = $Config->deviceByKey($k) || $Config->deviceByIP($k) || $Config->deviceByName($k);
      push(@$static_list, $dev) if ($dev);
   }
} elsif ( &add_inspector_data($Config, $static_list)) {
   Debug->print2("Doing " . ($#$static_list+1) . " device(s) found by the inspectors!");
   $DO_ALL_DEVICES = 1;

} elsif ($inspectors_on) {           #  RUN INSPECTORS 
   my $broke_list = Rasagent->run_inspectors($LB, $SINFO, $Config);
   &add_neighbors($Config, $static_list, $broke_list);
}

#  CREATE A ROUND-ROBIN LIST TO MONITOR
#
if ($#$static_list < 0) {
   $DO_ALL_DEVICES = 1    if (!$SINFO->{last_run});
   $SINFO->{last_run}     = time;
   $scheduled_run = 1;
   my ($x, $cnt, @hosts_as_devices);
   push(@hosts_as_devices,   PDM::ConfigFile->hostAsDevice($renv));
   if (!$MASTER_LOC) {
     foreach my $h (@$hosts) {
       push(@hosts_as_devices, PDM::ConfigFile->hostAsDevice($h))
          if ($h->{dh} ne "Y");
     }
   }
   my ($robin, $map) = Robin->sync($SINFO, $devices, \@hosts_as_devices);
   foreach my $el (@$robin) {
      my($time, $k) = split(/\t/, $el);
      my $d = $map->{$k};

      if (!System->get_allDevices()) {
         next if (time < $time);
      }
      $d->{scheduled} = 1;
      push(@$static_list, $d);
      last if (++$cnt > $MAX_DEVICES);
   }
   $map = undef;
}

my $fake;
if ($#$static_list < 0) {
  # used to force a run even with no devices to do.
  push(@$static_list, {active => "Y", type => "fake", key => "1"});
  $fake=1;
}
%CAT = ();
foreach my $d (@$static_list) {
  $CAT{$d->{type}} = 1;
}



######################################################
#  RUN THE AGENTS AGAINST static_list
######################################################

$renv->{"solution_prefix"} = Util->ipPrefix(System->ifconfig("dmfe1"));

if ($scheduled_run) {
  if (!$TESTMODE) {
     &fix_hostname($renv, $devices, $hosts, $notifs);
     use Daemons;
     Daemons->restart();
  }
  &clear_data_host($renv);
  &request_data_flush();
  #Discover->process_discovery();
  require Service;
  Service->clean_core(5);
}

print STDERR $LB->expand('running', $CONF, $today, "$s1 $HOST_NAME") . "\n";

# LOOK FOR CONFLICTS WITH DIAGS.
#
my($st_found);
if (!$FORCE) {
  if (($st_found= Agent->stor_conflicts()) ) {
     Debug->print2($LB->expand('diag_conflict',  $st_found));
  }
}
System->set_stFound($st_found);

&alternate_master_check($LB, $renv, $devices, $hosts, $notifs) if ($scheduled_run);

if (Repeat->timer("agent", 2*60)) {  # hourly operations
   TO->write_wwn_map();
   Scheduler->cleanUp(2); # remove 2 days old tests
   &rotate_logs();
   RasDB->REORG() if ($scheduled_run);
}

&check_locks($renv, $devices);

Report->copyToOld() if ($renv->{solution} ne "N"); # used by SAE.pm

my $agent_list = &list_agents($pdm, $renv);

use Health::GUI;
Health::GUI->all_logic();   # events queued by the GUI

my( @OUT, $ras_flag, %LOADED, %DYNDEV, $skip);

$DB::single=1;
if ($renv->{solution} ne "N") {
  require Agent::SP;
  $skip = Agent::SP->sequencer_status();  # check if the rack is off
  Debug->print2("PowerSequencer: skipping all modules because the rack is off (/opt/se6x20/tools/checkPower) ") if ($skip);

}

# DATAHOST, HOST, SAN, 

my ($loop);
for ($loop = 1; $loop <= 5; $loop++) {
    last if ($#$static_list < 0);
    Util->clear_run_cache();
    Health->reset_serial();
    foreach my $d (@$static_list) {
        delete $d->{problems};
    }
    if (!$skip) {
      if ($loop == 1 && $fake) {
        Debug->print2("Pass $loop: monitoring 0 devices (use rasagent -r) ...");
      } else {
        Debug->print2("Pass $loop: monitoring " . ($#$static_list+1) . " devices...");
      }
    }
    if ($loop > 1) {
      foreach my $dd (@$static_list) {
        Debug->print2("      : -> $dd->{name}/$dd->{ipno}");
      }
    }
    my $processed_devices = [];
  
    foreach my $modname (@$agent_list) {
       next if ($skip);
       Util->saveLastCommand("Running in module $modname\n");
       my($next, $dev_list) = &process_module($static_list, $st_found, 
                                              $modname, $LB, $loop+$DO_ALL_DEVICES);
       next if ($next);
       my $broke;
       $SINFO_DB->Lock();
       foreach my $d (@$dev_list) {
          my $sinfo_dev = $SINFO->{$d->{key}} || {};
          my $runc = $sinfo_dev->{run_count} ;
          if ($d->{problems} == 2) { # don't do those devices too often
             my $brk = Rasagent->inspectorListAdd($d, $SINFO, $processed_devices);
             $broke += $brk if ($runc);
          } else {
             $broke++ if ($d->{problems} && $runc);
             Rasagent->listAdd($processed_devices, [$d]);
             $sinfo_dev->{scheduled_run_count}++ if ($d->{scheduled});
          }
          $sinfo_dev->{run_count}++;
          $SINFO->{$d->{key}} = $sinfo_dev;
       }
       $SINFO_DB->UnLock();

       # on the first run, stop doing agents on the first problem 
       # if the list does not come from inspectors.

       if ($loop+$DO_ALL_DEVICES == 1 && $broke) {
          Debug->print2("Agent $modname found problems: aborting normal monitoring sequence..");
          last;
       }
    }
  
    # USE 'PROCESSED_DEVICES TO FILL 'STATIC_LIST WITH NEIGHBORS AND INSPECTOR FINDS.
    # IF ENTRIES in 'PROCESSED_DEVICES ARE BROKE....
    #
    $static_list = [];  
    my $broke_list;
    foreach my $d (@$processed_devices) {
       Robin->add($SINFO_DB, $d);
       if ($d->{problems} && $DYNDEV{$d->{key}}++ < 2) {# 1= already processed, add neighbors, 
                                                       # 2= broke but not processed.
         $broke_list++;
         if ($d->{problems} == 2) {
           Debug->print2("$d->{type}:$d->{name}/$d->{ipno}: adding device '$d->{name}/$d->{ipno}' to dynamic monitoring list..");
           push(@$static_list, $d);
         }
         my $neighbors_list = Rasagent->find_neighbors($Config, $d);
         foreach my $n (@$neighbors_list) {
           if (Rasagent->listAdd($static_list, [$n])) {
             Debug->print2("$d->{type}:$d->{name}/$d->{ipno}: ADDING neighbor '$n->{name}/$n->{ipno}' to dynamic monitoring list..");
           }
         }
       }
    }
    $broke_list += &add_inspector_data($Config, $static_list);
    last if ($broke_list == 0);
}

use Health::Host;
Health::Host->all_logic();

require Agent::T3;
Agent::T3->slice_map() if ($scheduled_run);
  
$DB::single = 1;

my $eventid_list;

# SAVE RACKS EVENTS LOCALLY.

FSA->RUN() if (!$MASTER_LOC);  # change events and create new events

PDM->saveStateLog();           # need to call before SE_modules, they read the ALARMS.

foreach my $modname ( 'SE_MODULE', 'SE2_MODULE', 'MASTER_MODULE') {
    my($next, $dev_list) = &process_module([], $st_found, $modname, $LB, 1);
}

PDM->saveStateLog();           # write to State.db and Events.log

State->sev_summary();

$pdm->storeMessages("DS") if ($renv->{solution} ne "N");

$eventid_list = PDM->getEventIdList(); 
Debug->print2(($#$eventid_list+1) . " new Events");

if ($MASTER_LOC) {  # a slave
  $pdm->storeMessages();
  PDM->push($MASTER_LOC);

} else {
   Debug->print2("\n" . $LB->{exe_pro});
   my $providers = Modules->load("Provider");
   foreach my $p (@$providers) {
      my $pro = "Provider::$p";
      my $f = $pro->new($pdm);
      
      if ($f->can('RUN')) {
         Util->saveLastCommand("Running provider $p\n");
         Debug->print1("\nProvider $p ");
         $f->RUN($eventid_list);
      } else {
         Debug->err('RUN_MISSING', ref($pro) );   
      }
   }
}
Util->saveLastCommand("All providers done\n");
PDM->deleteMessages($eventid_list);
Snapshot->process($renv) if ($#$eventid_list >= 0);

$pdm->serialize();
Timelapse->serialize();

Report->pushReportsList($MASTER_LOC, $force_refresh);

if ($MASTER_LOC) {  # a slave
  State->push($MASTER_LOC, $force_refresh);
}
use RasScheduler;
Util->saveLastCommand("Call RasScheduler\n");
RasScheduler->run();
Util->saveLastCommand("RasScheduler finished\n");

if ($scheduled_run && $loop == 1) { # clean run
   Debug->print2("Clean run of the agent.\n");
   delete $SINFO->{inspector_ran};
   &finish();

} else {
   &finish();
}


Util->saveLastCommand("Agent finished\n");
##########################################################
#                  SUBROUTINES
##########################################################

sub add_neighbors {
   my($Config, $static_list, $broke_list) = @_;

   if ($#$broke_list >= 0) {
     push(@$static_list, @$broke_list);
     foreach my $broke (@$broke_list) {
       my $neighbors_list = Rasagent->find_neighbors($Config, $broke);
       Rasagent->listAdd($static_list, $neighbors_list, "Neighbors");
     }   
   }
}


sub add_inspector_data {
  my($Config, $static_list) = @_;
  $SINFO_DB->Lock();
  my $cnt;
  my $broke_list = [];
  if (exists $SINFO->{inspector_data}) {
     $broke_list =  $SINFO->{inspector_data};
     delete  $SINFO->{inspector_data};
     $cnt++;
  }
  $SINFO_DB->UnLock();
  &add_neighbors($Config, $static_list, $broke_list);
  return $cnt;
}



sub request_data_flush {
  my $F = System->get_home() . "/DATA/request_data_flush";
  if (-f $F) {
     unlink $F;
     my $D = System->get_home() . "/DATA/OLD_REPORTS";
     opendir(O, $D);
     my @L = readdir(O);
     foreach my $file (@L) {
        if ($file =~ /.+\:.+/) {
           unlink "$D/$file";
        }
     }
  }
}



sub process_module {
  my($static_list, $st_found, $modname, $LB, $loop) = @_;

  return(1,[]) if (Solution->exclude($renv->{solution}, $modname));

  if ($st_found && index("T3MESSAGE,MESSAGE,DSPMESSAGE,TRAPMESSAGE", $modname) < 0 ) {
     if ($CAT{lc($modname)}) {
        Debug->print2("\n" . $LB->expand('exe_agent', $modname));
        Debug->print2($LB->{diag_skip});
     }
     return (1,[]);
  }
  if (!Agent->category_selected($modname) && index(",MASTER_MODULE,SE_MODULE,SE2_MODULE,SLAVE,TOPO,", $modname) < 0) {
     # Debug->print2( $LB->{skip} . " $modname.") if ($loop == 1);
     return (1,[]); 
  }
  if ($CAT{lc($modname)}) {
    Debug->print1("Executing agent $modname");
  }
  return Rasagent->process_module0($static_list, $modname, $LB, $loop, $SINFO);
}


#  $loop = 0 for INSPECTORS

  

# at least run the T3MESSAGE module.

sub inspectors {
    return [];
}


sub finish {
   Process->done("rasagent");
   Util->set_Xmins("frequency.cache");
   Debug->print(PID => $LB->{remove_pid} . "(REMOVE_PID): AGENT EXITING!");
   unlink $PIDfile;
}
  

sub usage {
  print "
  usage: rasagent 
   -d <level> : Debug (2=most trace, 3=more details, write snapshot.log in /DATA)
              : -d2 skips the round-robin and does all devices (up to 100).
   -r         : ignore round-robin , do all devices.
   -a         : do all devices even when one is broke.
   -f         : Force, run all agents, even if a diag. is running.
   -l [list]  : process devices list (device_keys separated by ':')
   -I         : Run Inspectors first (if there is no pending problems).
   -M         : Clear Email/Events/Thresholds maximum database.
   -A         : audit all devices, including revision check.
   -T[snapid] : test mode (use DATA/TEST_REPORTS, use snapid as directory if present.
   -b         : snapshots base directory (default is /var/opt/SUNWstade/SNAP).

   -? -h      : Help
   -v         : version
   -c         : started from the cron.
   -C         : clear CACHE.db and MAXEVENTS.db
    
";
}

sub fix_hostname {
  my($renv, $devices, $hosts, $notifs) = @_;

  my $sys_hostname = System->hostname();
  return if ($renv->{dont_fix_hostname});

  if ($renv->{solution} ne "N" && $renv->{hostname} ne $sys_hostname ) {
     Debug->print2("The hostname of this Sun-Solution $renv->{solution_model} ".
                   "changed from '$renv->{hostname} to '$sys_hostname");
     my $D = System->get_home() . "/DATA/";
     unlink "$D/REMOTE_LISTS/$renv->{hostname}";
     rename "$D/Proc/$renv->{hostname}" , "$D/Proc/$sys_hostname";
     rename "$D/state/$renv->{hostname}", "$D/state/$sys_hostname";
     my $lh = length($renv->{hostname});
     my $short = Util->shortHostname($sys_hostname);
     my $old   = $renv->{hostname};
     foreach my $d (@$devices) {
        $d->{name} =~ s/$old/$short/;
     }
     unlink "$D/topo/$renv->{hostname}";
     unlink "$D/OLD_REPORTS/sp:$renv->{hostname}";
     unlink "$D/cache_st_last";
     unlink "$D/CACHE/HostInfo"; # clear for Agent->hostInfo();
  
     $renv->{hostname} = $sys_hostname;
     PDM::ConfigFile->write($renv, $devices, $hosts, $notifs);
  
     require TO;
     require Events;
     TO->clearTopo($sys_hostname);
     my $to = TO->readTopo($sys_hostname,1);
     TO->copyTopo($sys_hostname,"MERGE-MASTER");
  }
}


sub auto_discovery {
  my($renv) = @_;

  if ($renv->{auto_discover} eq "auto") {
    my $timer = Timer->isXdays("auto_discover", 1);
    if ($timer eq "YES") {
      Debug->print2("AutoDiscovery from deviceIP.conf");
      require Logic::SWITCH;
      my($err, $nodes, $info) = Logic::SWITCH->addFromFile({
                                host => "", TEXT => 1, silent => 1,
                                monitor_on => 1,
                                fileToUse => "/etc/deviceIP.conf",
                                select_host => "local" } );
      foreach my $n (@$nodes) {
         Debug->print2("Adding $n->{ipno}");
      }
    }
  }
}

sub alternate_master_check {
  my($LB, $renv, $devices, $hosts, $notifs) = @_;

  $renv->{role} = "M" if (!-f System->get_home() . "/DATA/MASTER" && !$renv->{role});
  
  if ($renv->{role} eq "AM") {
     my $active = Util->file2string("ALTACTIVE");
     my $M = $MASTER_LOC || $active;
     my $alive;
     if (!$M) {
       Debug->err(TEXT => $LB->{no_master});
       exit(1);
     } else {
       $alive = Util->ping($M);
     }
     my $new_master;
  
     if (!$alive &&  !$active) { # take over
        Debug->print2($LB->expand('cannot_ping', $MASTER_LOC));
  
        $new_master = Util->name2ip($renv->{hostname}) || $renv->{hostname};
        Events->saveEvent("AM+",$renv->{hostname}, $MASTER_LOC);
  
        Util->string2file($MASTER_LOC, "ALTACTIVE");
        Util->clearMaster();
        $MASTER_LOC = undef;
        foreach my $d (@$devices) {  # fix the host on the devices
            $d->{host} = "" if ($d->{host} eq $renv->{hostname});
        }
  
     } elsif ($alive && $active) {  # release
        $new_master = $active;
        my $info = Util::Http->getCommand($new_master,"INFO1", 10);
        if ($info =~ /version/) {
          Events->saveEvent("AM-", $renv->{hostname} , $active);
  
          Util->setMaster($new_master);
          unlink System->get_home() . "/DATA/ALTACTIVE";
          $MASTER_LOC = $new_master;
          Debug->print2("Can ping $M, giving-up master...");
        }
  
     }
     if ($new_master) {
        foreach my $h (@$hosts) {
          next if ($h->{role} eq "AM" || $h->{role} eq "M");
          my $ip = $h->{ipno} || Util->name2ip($h->{hostname});
          my($err, $ans) = Util::Http->saveFile($ip, "MASTER", $new_master, 20);
        }
        PDM::ConfigFile->write($renv, $devices, $hosts, $notifs);
     }
  }
}

sub clear_data_host {
  my($renv) = @_;
  my $D = "$HOME/DATA/DataHost";

  if (index($renv->{categories}, "datahost") < 0) {
     opendir(W, $D);
     my @files = readdir(W); closedir(W);
     foreach my $f (@files) {
        if ($f =~ /loglines/) {
           unlink "$D/$f";
        }
     }
  }
}



sub rotate_logs {
  System->rotate("$HOME/log/inspect.log");
  System->rotate("$HOME/log/file_transfer.log", 1, 1);
  System->rotate("$HOME/log/agent.log");
  System->rotate("$HOME/log/rashstart.log");
  System->rotate("$HOME/log/cron.log");
  System->rotate("$HOME/log/volverify.log");
  System->rotate("$HOME/log/patch_scheduler.log");
  System->rotate("$HOME/log/cron_patch.log");
  System->rotate("$HOME/log/errors.log");
  System->rotate("$HOME/log/info.log");
  System->rotate("$HOME/log/snmpagent.log");
  System->rotate("$HOME/log/scheduler.log");
  System->rotate("$HOME/DATA/Events.log");
  System->rotate("$HOME/DATA/EventsSys.log");
  System->rotate("$HOME/DATA/Notifications.log");
  System->rotate("/var/adm/trap_messages");
  Util->clean_run_commands();
}


sub check_locks {
  my($renv, $devices) = @_;

  my $SE_CFG = "/var/adm/log/SEcfglog";
  my $list;

  if (-f $SE_CFG) {  # INDY
    my ($enter, $exit, $enter_command, $enter_list);
    my (%H, $l);
    if (open(O, "/bin/ls /opt/SUNWsecfg/etc/.[stv]*lock 2>/dev/null|")) {
      while ($l = <O>) {
        $enter_list .= substr($l, 5, -6) . ",";
      }
      close(O);
    }
    if ($enter_list) {
      my $now = Util->get_today(); 
      Debug->print2("SE_CFG: Locks found on $enter_list");
      $list = Util->shortHostname($renv->{hostname}) . "-" . $enter_list;
    }
  }

  if ($renv->{solution} eq "se2") {
    my %MAP;
    foreach my $d (@$devices) {
      my $n = rindex($d->{name}, "-");
      my $n2 = $n > 0 ? substr($d->{name}, $n+1) : $d->{name};
      $MAP{$n2} = $d->{name};
    }
    opendir(D, "/tmp");
    my @F = readdir(D); closedir(D);

    foreach my $f (@F) {
       next if (!$MAP{$f});
       my $age = Util->getFileAge("/tmp/$f", 'h');
       open(OO, $f);
       my $exp = 24;
       my @L = <OO>; close(OO);
       if ("@L" =~ /Exp\w*: (\d+)/) {
           $exp = $1;
       }
       if ($age > $exp) {  # prune.
          unlink "/tmp/$f";
       } else {            # skip this device.
          $list .= "$MAP{$f},";
       }
    }
  }
  if ($list) {
    # return a list of devices that are locked
    Debug->print2("Found locked devices: $list");
    System->set_se_conflict($list); # skip this component;
  }
}
  
sub list_agents {
  my($pdm, $renv) = @_;
  my(@mess, @san, @special, @other, @all, @nowhere);

  my $mods      = Modules->read("Agent");
  my $skip_host = $renv->{solution} eq "N" ? "SP" : "HOST";

# push topo and san at the end of the agent list
# message at the beginning for early warnings

  foreach my $m (sort @$mods) {
     next if ($m =~ /Parent/ || $m eq $skip_host);
     if (index(",PING,",",$m,") >= 0) {

     } elsif ($m eq "EVENTS" || $m eq "SLAVE" || index($m, "MESSAGE") >= 0) {
        push(@mess, $m);

     } elsif (index(",MASTER_MODULE,SE_MODULE,SE2_MODULE,", ",$m,") >= 0) {
        push(@nowhere, $m);

     } elsif (index(",TOPO,HOST,SP,", ",$m,") >= 0) {
        push(@special, $m);

     } elsif (index(",SAN,", ",$m,") >= 0) {
        push(@san, $m);

     } else {
        push(@other, $m);
     }
  }
  push(@all, @mess);
  push(@all, @other);
  push(@all, @special);
  push(@all, @san);

  return \@all;
}

sub version {
  my($v) = `/usr/bin/cat /opt/SUNWstade/System/config"`;
  print "
  Storage Automated Diagnostic Environement $v \n";
  exit;

}




sub datetime {
  my(@date) = localtime(time); $date[4]++; $date[5] += 1900;
  sprintf("%2.2d-%2.2d-%d %2.2d:%2.2d:%2.2d",
           $date[4],$date[3],$date[5],$date[2],$date[1],$date[0]);
}

#######################
#     SIGNALS
#   sub die_signal {
#     my($sig) = @_;
#     print "Caught a signal SIG$sig -- aborting \n";
#     exit(1);
#   }
#   
#   sub catch_signal {
#     my($sig) = @_;
#     Debug->print3("Caught a signal SIG$sig") if ($sig ne "CHLD");
#   }
#   
#   $SIG{CHLD} = \&catch_signal; # 'IGNORE';
#   $SIG{TERM} = \&catch_signal;
#   $SIG{INT}  = \&die_signal;
#   
#   foreach my $sig ('SEGV', 'QUIT','INT','ILL','TRAP','IOT','ABRT',
#                     'EMT','FPE','KILL','BUS') {
#       $SIG{$sig} = \&die_signal;
#   }
