#!/usr/local/bin/perl

#
# Check a Dell iDRAC service processor
#
# $Header: /home/doke/work/nagios/RCS/check_dell_idrac,v 1.4 2016/05/25 01:12:43 doke Exp $



use strict;
use warnings;
use Getopt::Long;
use Expect;
use Data::Dumper;

use vars qw( $host $username $passwordfile $interface $timeout $verbose $help
    @crits @warns @unknowns @ignores @oks $rc $sep $password $do_idrac7 $do_idrac8 );

$host = undef;
$username = 'root';
$passwordfile = '/usr/local/nagios/etc/service_processor.pw';
$interface = 'lanplus';
$do_idrac7 = 0;
$do_idrac8 = 0;
$timeout = 10;

$ENV{PATH}='/usr/local/bin:/opt/sfw/bin:/usr/sfw/bin:/usr/bin:/bin:/usr/sbin:/sbin';

$verbose = 0;
$help = 0;


sub usage {
    my( $rc ) = @_;
    print "Usage: $0 [-vh] [-H host] [-U username] [-f passwordfile]
    -H s  hostname
    -U s  username on service processor [$username]
    -f s  file containing password [$passwordfile]
    -I s  interface: bmc, open, lan, lanplus [$interface]
    -7    use extra iDRAC7 tests
    -8    use extra iDRAC8 tests
    -v    verbose
    -h    help
";
    exit $rc;
    }

Getopt::Long::Configure ("bundling");
GetOptions(
    'H=s' => \$host,
    'U=s' => \$username,
    'f=s' => \$passwordfile,
    'I=s' => \$interface,
    '7' => \$do_idrac7,
    '8' => \$do_idrac8,
    'v+' => \$verbose,
    'h' => \$help,
    );
&usage( 0 ) if ( $help );

&usage( 0 ) if ( ! $host );

if ( $do_idrac8 )  { 
    $do_idrac7 = 1;
    }
if ( $do_idrac8 && $interface eq 'lan' )  { 
    $interface = 'lanplus'; 
    }

#&check_ipmi();
&check_racadm();

$rc = 0;
$sep = '';
if ( $#crits >= 0 ) {
    $rc = 2;
    print "CRITICAL ", join( ", ", @crits );
    $sep = '; ';
    }
if ( $#warns >= 0 ) {
    $rc = 1 if ( $rc == 0 );
    print $sep, "Warning ", join( ", ", @warns );
    $sep = '; ';
    }
if ( $#unknowns >= 0 ) {
    $rc = -1 if ( $rc == 0 );
    print $sep, "Unknown ", join( ", ", @unknowns );
    $sep = '; ';
    }
if ( $rc == 0 || $verbose ) {
    print $sep, "Ok ", join( ", ", @oks );
    $sep = '; ';
    }
if ( $#ignores >= 0 ) {
    print $sep, "Ignoring ", join( ", ", @ignores );
    $sep = '; ';
    }
print "\n";
exit $rc;




##################


sub check_ipmi {
    my( $ipmicmd, $dev, $sdr_cache, $cmd, $nok, $key, $val, $descr, $junk,
	%frus, $fru, $mfg, $model );

    $cmd = "ipmitool -V > /dev/null";
    $verbose && print "+ $cmd\n";
    system( $cmd );
    if ( ( $? >> 8 ) != 0 ) {
	push @unknowns, "unable to run ipmitool: $!";
	return;
	}

    if ( $host ) {
	# remote host
	# first make sure we can ping it.
	# ipmitool has some really long timeouts
	if ( ! ping( $host ) ) {
	    push @unknowns, "can't ping $host";
	    return;
	    }

	if ( $interface eq 'auto' || $interface eq 'bmc' ) {
	    $interface = 'lan';
	    }

	$ipmicmd = "ipmitool -I $interface -H '$host'";
	$ipmicmd .= " -U '$username'" if $username;
	$ipmicmd .= " -f '$passwordfile'" if $passwordfile;

	# In ipmitool 1.8.8, the -c option for csv seperated output is very broken
	# It runs lines together.

	$sdr_cache = "/tmp/sdr_cache.$host.$<";
	if ( ! ( -f $sdr_cache ) || ( -M _ ) > 1.0 ) {
	    $cmd = "$ipmicmd sdr dump $sdr_cache > /dev/null";
	    $verbose && print "+ $cmd\n";
	    system( $cmd );
	    if ( ( $? >> 8 ) == 0 ) {
		$ipmicmd .= " -S '$sdr_cache'";
		}
	    else {
		push @warns, "unable to create sdr cache";
		}
	    }
	}
    else {
	# no host, we're local

	if ( $interface eq 'auto' ) {
	    if ( -c "/dev/bmc" ) {
		$dev = "/dev/bmc";
		$interface = 'bmc';
		}
	    elsif ( -c "/dev/ipmi0" ) {
		$dev = "/dev/ipmi0";
		$interface = "open";
		}
	    elsif ( -c "/dev/ipmi/0" ) {
		$dev = "/dev/ipmi/0";
		$interface = "open";
		}
	    elsif ( -c "/dev/ipmidev/0" ) {
		$dev = "/dev/ipmidev/0";
		$interface = "open";
		}
	    else {
		push @unknowns, "can't find bmc or ipmi device";
		return;
		}
	    }

	if ( -w $dev )  {
	    $ipmicmd = "ipmitool -I $interface";
	    }
	else {
	    $ipmicmd = "sudo -n ipmitool -I $interface";
	    }
	}


    $cmd = "$ipmicmd chassis status";
    $verbose && print "+ $cmd\n";
    if ( ! open( pH, "$cmd |" ) ) {
	push @unknowns, "can't run $cmd: $!";
	return;
	}
    while( <pH> ) {
	$verbose && print;
	chomp;
	( $key, $val ) = split( m/\s*:\s*/, $_, 2 );
	if ( $key eq 'System Power' && $val ne 'on' ) {
	    # wtf?  So how is this program running?
	    push @crits, "$key: $val";
	    }
	elsif ( $key eq 'Power Restore Policy' && $val !~ m/always-on|previous/i ) {
	    push @ignores, "$key: $val";
	    }
	elsif ( $key =~ m/Main Power Fault/i && $val =~ m/true/i ) {
	    # sometimes this gets left on after the problem clears
	    # so only report it if there are other problems
	    # we've seen this on ironchef, an X4600 with ilom v2.0.2.5
	    push @ignores, "main power fault $val";
	    }
	elsif ( $key =~ m/fault|lockout|overload/i && $val !~ m/false|inactive/i ) {
	    push @warns, "$key: $val";
	    }
	}
    close pH;


    $frus{ 'mainboard' }{ 'product_manufacturer' } = '';

    $cmd = "$ipmicmd fru list";
    $verbose && print "+ $cmd\n";
    if ( ! open( pH, "$cmd 2>&1 |" ) ) {
	push @unknowns, "can't run $cmd: $!";
	return;
	}
    while( <pH> ) {
	$verbose && print;
	chomp;
	next unless ( m/\s*(\S*?)\s*:\s*(\S.*)\s*$/ );
	( $key, $val ) = split( m/\s*:\s*/, $_, 2 );
	$verbose && print "key $key, val $val\n";
	if ( $key =~ m/FRU Device Description/i ) {
	    $fru = $val;
	    $verbose && print "fru $fru\n";
	    }
	elsif ( $key =~ m/Product Manufacturer/i ) {
	    $verbose && print "fru $fru, product_manufacturer $val\n";
	    $frus{ $fru }{ 'product_manufacturer' } = $val;
	    }
	elsif ( $key =~ m/Product Name/i ) {
	    $verbose && print "fru $fru, product_name $val\n";
	    $frus{ $fru }{ 'product_name' } = $val;
	    }
	elsif ( $key =~ m/Product Serial/i ) {
	    $frus{ $fru }{ 'product_serial' } = $val;
	    }
	elsif ( $key =~ m/Chassis Serial/i ) {
	    $frus{ $fru }{ 'chassis_serial' } = $val;
	    }
	elsif ( $key =~ m/Board Mfg/i ) {
	    $frus{ $fru }{ 'board_mfg' } = $val;
	    }
	elsif ( $key =~ m/Board Product/i ) {
	    $frus{ $fru }{ 'board_product' } = $val;
	    }
	elsif ( $key =~ m/Board Serial/i ) {
	    $frus{ $fru }{ 'board_serial' } = $val;
	    }
	elsif ( $verbose > 1 )  {
	    print "unparsed: $_\n";
	    }
	}
    close pH;


    $mfg = '';
    $model = '';

    # sun style, works on SUN FIRE X4600, SPARC T3-1, Sun Fire X2200 M2
    foreach $fru ( keys %frus ) {
	if ( $fru =~ m%Mainboard|^/SYS \(ID%i ) {
	    foreach $key ( keys %{$frus{ $fru }} ) {
		$val = $frus{ $fru }{ $key };
		$verbose && print "fru $fru, key $key, val $val, mfg $mfg, model $model\n";
		if ( ! $mfg && $key =~ m/Product.Manufacturer/i ) {
		    $mfg = $val;
		    }
		elsif ( ! $model && $key =~ m/Product.Name/i ) {
		    $model = $val;
		    }
		}
	    }
	}
    if ( ! $mfg || ! $model ) {
	# Dell style, works on PowerEdge R515, PowerEdge R720
	foreach $fru ( keys %frus ) {
	    if ( $fru =~ m/Builtin FRU Device/i ) {
		foreach $key ( keys %{$frus{ $fru }} ) {
		    $val = $frus{ $fru }{ $key };
		    $verbose && print "fru $fru, key $key, val $val, mfg $mfg, model $model\n";
		    if ( ! $mfg && $key =~ m/Board.Mfg/i ) {
			$mfg = $val;
			}
		    elsif ( ! $model && $key =~ m/Board.Product/i ) {
			$model = $val;
			}
		    }
		}
	    }
	}

    $verbose && print "mfg $mfg, model $model\n";



    $cmd = "$ipmicmd sdr elist all";
    $verbose && print "+ $cmd\n";
    if ( ! open( pH, "$cmd |" ) ) {
	push @unknowns, "can't run $cmd: $!";
	return;
	}
    $nok = 0;
    while( <pH> ) {
	$verbose && print;
	chomp;
	( $key, $junk, $val, $junk, $descr ) = split( m/\s*\|\s*/, $_, 5 );
	if ( $val eq 'ok' ) {
	    $nok++;
	    }
	# I have no idea what these values mean, but they seem to be useless
	elsif ( $val eq 'ns' || $val eq 'lcr' || $val eq 'lnc' ) {
	    # ignore it
	    }
	elsif ( $descr =~ '0 unspecified' ) {
	    # ignore it
	    }
	else {
	    push @crits, "$key $val $descr";
	    }
	}
    push @oks, "$nok ok sensors";
    close pH;


    if ( $mfg =~ m/Sun/i ) {
	# sunoem commands need to be run as root, no idea why
	if ( $ipmicmd =~ m/^sudo / ) {
	    $cmd = "$ipmicmd sunoem led get";
	    }
	else {
	    $cmd = "sudo -n $ipmicmd sunoem led get";
	    }
	$verbose && print "+ $cmd\n";
	if ( ! open( pH, "$cmd 2>&1 |" ) ) {
	    push @unknowns, "can't run $cmd: $!";
	    return;
	    }
	$nok = 0;
	while( <pH> ) {
	    $verbose && print;
	    chomp;
	    next if ( m/command failed/i );
	    ( $key, $val ) = split( m/\s*\|\s*/, $_, 2 );
	    if ( ( $key eq 'sys.power.led'
			|| $key eq 'bp.power.led'    # specific to Sun X4100?
			|| $key eq 'fp.power.led'    # specific to Sun X4100?
			|| $key =~ '(ft[0-4]|sc)\.act\.led' )  # specific to Sun X4500?
		    && $val eq 'ON' ) {
		$nok++;
		}
	    elsif ( $key eq 'sys.locate.led' ) {
		# don't care about the locating led
		$nok++;
		next;
		}
	    elsif ( $val eq 'OFF' ) {
		$nok++;
		next;
		}
	    elsif ( $key eq 'bmc_send_cmd: Permission denied' ) {
		# ignore it
		}
	    else {
		push @crits, "$key $val";
		}
	    }

	if ( $nok ) {
	    push @oks, "$nok ok leds";
	    }
	else {
	    # lots of systems produce no led output, all x2200s?
	    #push @unknowns, "no leds could be checked";
	    }
	close pH;
	}
    #elsif ( $mfg =~ m/Dell/i ) {
	# There's a delloem command, but so far I havn't found anything
	# useful to monitor in it.  You can see the mac address, or set the
	# led states, but you can't get the led states.
	#}

    if ( $mfg ) {
	push @oks, $mfg;
	}
    if ( $model ) {
	push @oks, $model;
	}

    }







##################






sub check_racadm {

    $password = get_password( $passwordfile );

    # getsvctag
    # getsysinfo
    # racdump
    # getsel
    # getconfig ...

    if ( $do_idrac7 ) { 
	check_racadm_sensors();
	}
    }




sub check_racadm_sensors {
    my( $buf, $sensor_type, $sensor, $ps, $fan, $cpu, $slot, $status,
	$state, $reading, $intrusion, $type, $lower_critical,
	$upper_critical, $lower_non_critical, $upper_non_critical, $units,
	$nps, $ntemps, $ncpus, $ndimms, $nfans, $nvoltages, $ncurrents );

    $buf = get_racadm_cmd( 'getsensorinfo' );
    $sensor_type = '';
    foreach ( split( m/\n/, $buf ) ) {
	$verbose && print "< $_\n";

	if ( m/^ \s* Sensor \s+ Type \s* : \s* (\S.*+) \s*$/ix ) {
	    $sensor_type = uc $1;
	    $verbose && print "> sensor type $sensor_type\n";
	    }

	elsif ( m/^<Sensor/i ) {
	    next;
	    }
	elsif ( m/^\[Key/i ) {
	    next;
	    }

	#Sensor Type : POWER
	#<Sensor Name>                   <Status>                 <Type>
	#PS1 Status                      Present                  AC
	elsif ( $sensor_type eq 'POWER'
		&& m/(PS\d+) \s+ Status \s+ (\S+) \s+ (\S+) /xi ) {
	    $ps = $1;
	    $status = $2;
	    $type = $3;
	    $verbose && print "> power $ps $status $type\n";
	    $nps++;
	    if ( $status ne 'Present' ) {
		push @warns, "$ps $status $type";
		}
	    }

	#Sensor Type : TEMPERATURE
	#<Sensor Name>            <Status>    <Reading> <lc> <uc>  <lnc>[R/W]  <unc>[R/W]
	#[Key = iDRAC.Embedded.1#SystemBoardInletTemp]
	#System Board Inlet Temp       Ok      20C      -7C  47C     3C [Y]      42C [Y]
	elsif ( $sensor_type eq 'TEMPERATURE'
		&& m/(.*?) Temp \s+ (\S+) \s+ (\d+)C \s+ ([\d-]+)C \s+ (\d+)C \s+
		    ([\d-]+)C \s+ \[[YN]\] \s+ (\d+)C \s+ \[[YN]\]/xi ) {
	    $sensor = $1;
	    $status = $2;
	    $reading = $3;
	    $lower_critical = $4;
	    $upper_critical = $5;
	    $lower_non_critical = $6;
	    $upper_non_critical = $7;
	    $ntemps++;
	    $verbose && print "> temp $sensor $status $reading\n";
	    if ( $status eq 'Critical'
		    || $reading < $lower_critical || $upper_critical < $reading ) {
		push @crits, "$sensor temp $status ${reading}C";
		}
	    elsif ( $status ne 'Ok'
		    || $reading < $lower_non_critical || $upper_non_critical < $reading ) {
		push @warns, "$sensor temp $status ${reading}C";
		}
	    }

	#Sensor Type : FAN
	#<Sensor Name>                   <Status>    <Reading>   <lc>        <uc>
	#System Board Fan1 RPM           Ok          3000RPM             360RPM      NA
	elsif ( $sensor_type eq 'FAN'
		&& m/(.*?) RPM \s+ (\S+) \s+ (\d+)RPM \s+ (\d+)RPM \s+ (\S+) /xi ) {
	    $fan = $1;
	    $status = $2;
	    $reading = $3;
	    $lower_critical = $4;
	    $nfans++;
	    $verbose && print "> fan $fan $status $reading\n";
	    if ( $status eq 'Critical'
		    || $reading < $lower_critical ) {
		push @crits, "$fan RPM $status ${reading}RPM";
		}
	    elsif ( $status ne 'Ok' ) {
		push @warns, "$fan RPM $status ${reading}RPM";
		}
	    }

	#Sensor Type : VOLTAGE
	#<Sensor Name>                   <Status>    <Reading>   <lc>        <uc>
	#CPU1 VCORE PG                   Ok          Good                NA          NA
	#PS1 Voltage 1                   Ok          206V                NA          NA
	elsif ( $sensor_type eq 'VOLTAGE'
		&& m/(.*?) \s{2,40} (\S+) \s+ (Good|\d+V|\S+) \s+
		    (\S+) \s+ (\S+) \s*$/xi ) {
	    $sensor = $1;
	    $status = $2;
	    $reading = $3;
	    $lower_critical = $4;
	    $upper_critical = $5;
	    $nvoltages++;
	    $verbose && print "> voltage $sensor $status $reading\n";
	    if ( $status eq 'Critical' ) {
		push @crits, "$sensor $status $reading";
		}
	    elsif ( $status ne 'Ok' ) {
		push @warns, "$sensor $status $reading";
		}
	    }

	#Sensor Type : CURRENT
	#<Sensor Name>            <Status>    <Reading> <lc> <uc>  <lnc>[R/W]  <unc>[R/W]
	#[Key = iDRAC.Embedded.1#PS1Current1]
	#PS1 Current 1                 Ok      1.0Amps  NA   NA      0Amps [N]      0Amps [N]
	#System Board Pwr Consumption  Ok      196Watts NA   1974Watts0Watts [N]      896Watts [Y]
	# uc and lnc are run together?
	elsif ( $sensor_type eq 'CURRENT'
		&& m/(.*?) \s{2,40} (\S+) \s+ ([\d\.]+)(Amps|Watts) \s*
		    (NA|[\d\.]+(Amps|Watts)) \s* (NA|[\d\.]+(Amps|Watts)) \s*
		    (NA|[\d\.]+(Amps|Watts)) \s+ \[[YN]\] \s*
		    (NA|[\d\.]+(Amps|Watts)) \s+ \[[YN]\] /xi ) {
	    $sensor = $1;
	    $status = $2;
	    $reading = $3;
	    $units = $4;
	    $ncurrents++;
	    $verbose && print "> current $sensor $status $reading $units\n";
	    if ( $status eq 'Critical' ) {
		push @crits, "$sensor $status $reading$units";
		}
	    elsif ( $status ne 'Ok' ) {
		push @warns, "$sensor $status $reading$units";
		}
	    }


	#Sensor Type : PROCESSOR
	#<Sensor Name>            <Status>    <State>             <lc>        <uc>
	#CPU1 Status              Ok          Presence Detected   NA          NA
	elsif ( $sensor_type eq 'PROCESSOR'
		&& m/(CPU\d+) \s+ Status \s+ (\S+) \s+ (Presence \s+ Detected|\S+) \s/xi ) {
	    $cpu = $1;
	    $status = $2;
	    $state = $3;
	    $ncpus++;
	    $verbose && print "> cpu $cpu $status $state\n";
	    if ( $state eq 'Presence Detected' )  {
		if ( $status eq 'Critical' ) {
		    push @crits, "$cpu $status";
		    }
		elsif ( $status ne 'Ok' ) {
		    push @warns, "$cpu $status";
		    }
		}
	    }

	#Sensor Type : MEMORY
	#<Sensor Name>                   <Status>    <State>             <lc>        <uc>
	#DIMM A1        Ok          Presence Detected   NA          NA
	#DIMM A4        Warning     Presence Detected   NA          NA
	#DIMM A9        N/A         Absent              NA          NA
	elsif ( $sensor_type eq 'MEMORY'
		&& m/DIMM \s+ (\S+) \s{2,30} (\S+) \s{2,30} (Presence \s+ Detected|\S+) 
		    \s{2,30} (\S+) \s+ (\S+) \s*$/xi ) {
	    $slot = $1;
	    $status = $2;
	    $state = $3;
	    $ndimms++;
	    $verbose && print "> memory $slot $status $state\n";
	    if ( $state eq 'Presence Detected' )  {
		if ( $status eq 'Critical' ) {
		    push @crits, "DIMM $slot $status";
		    }
		elsif ( $status ne 'Ok' ) {
		    push @warns, "DIMM $slot $status";
		    }
		}
	    }

	#Sensor Type : BATTERY
	#<Sensor Name>                   <Status>    <Reading>   <lc>        <uc>
	#System Board CMOS Battery       Ok          Present             NA          NA
	elsif ( $sensor_type eq 'BATTERY'
		&& m/(.*?) \s{2,40} (\S+) \s+ (\S+) \s+ (\S+) \s+ (\S+) \s* $/xi ) {
	    $sensor = $1;
	    $status = $2;
	    $reading = $3;
	    $lower_critical = $4;
	    $upper_critical = $5;
	    $verbose && print "> battery $sensor $status $reading\n";
	    if ( $status eq 'Critical' ) {
		push @crits, "$sensor $status $reading";
		}
	    elsif ( $status ne 'Ok' || $reading ne 'Present' )  {
		push @warns, "$sensor $status $reading";
		}
	    }

	#Sensor Type : PERFORMANCE
	#<Sensor Name>                   <Status>    <State>        <lc>      <uc>
	#System Board Power Optimized    Ok          Not Degraded   NA        NA
	elsif ( $sensor_type eq 'PERFORMANCE'
		&& m/(.*?) \s{2,40} (\S+) \s+ (Not \s+ Degraded|\S+) \s+ 
		    (\S+) \s+ (\S+) \s* $/xi ) {
	    $sensor = $1;
	    $status = $2;
	    $state = $3;
	    $lower_critical = $4;
	    $upper_critical = $5;
	    $verbose && print "> performance $sensor $status $state\n";
	    if ( $status eq 'Critical' ) {
		push @crits, "$sensor $status $state";
		}
	    elsif ( $status ne 'Ok' || $state ne 'Not Degraded' )  {
		push @warns, "$sensor $status $state";
		}
	    }

	#Sensor Type : INTRUSION
	#<Sensor Name>                   <Intrusion>    <Status>
	#System Board Intrusion          Closed         Power ON
	elsif ( $sensor_type eq 'INTRUSION'
		&& m/(.*?) \s{2,40} (\S+) \s+ (Power \s+ ON|\S+) \s* $/xi ) {
	    $sensor = $1;
	    $intrusion = $2;
	    $status = $3;
	    $verbose && print "> intrusion $sensor $intrusion $status\n";
	    if ( $intrusion ne 'Closed' || $status ne 'Power ON' )  {
		push @warns, "$sensor $intrusion $status";
		}
	    }

	#Sensor Type : REDUNDANCY
	#<Sensor Name>                   <Status>                 <Type>
	#System Board Fan Redundancy     Full Redundant           Fan
	#System Board PS Redundancy      Full Redundant           PSU
	#IDSDM Redundancy                Full Redundant           Unknown
	elsif ( $sensor_type eq 'REDUNDANCY'
		&& m/(.*?) \s{2,40} (Full \s+ Redundant|\S+) \s{2,40} (\S+) \s* $/xi ) {
	    $sensor = $1;
	    $status = $2;
	    $type = $3;
	    $verbose && print "> redundancy $sensor $status $type\n";
	    if ( $status ne 'Full Redundant' )  {
		push @warns, "$sensor $status $type";
		}
	    }

	#Sensor Type : IDSDM
	#<Sensor Name>                   <Status>
	#IDSDM SD1                       Good
	#IDSDM SD2                       Good
	elsif ( $sensor_type eq 'IDSDM'
		&& m/(.*?) \s{2,40} (\S+) \s* $/xi ) {
	    $sensor = $1;
	    $status = $2;
	    $verbose && print "> idsdm $sensor $status\n";
	    if ( $status ne 'Good' )  {
		push @warns, "$sensor $status";
		}
	    }

	#Sensor Type : SYSTEM PERFORMANCE
	#<Sensor Name>            <Status>    <Reading> <lc> <uc>  <lnc>[R/W]  <unc>[R/W]
	#[Key = iDRAC.Embedded.1#SystemBoardCPUUsage]
	#System Board CPU Usage   Ok             1%      NA  NA    NA  [N]      99% [Y]
	#[Key = iDRAC.Embedded.1#SystemBoardIOUsage]
	#System Board IO Usage    Ok             0%      NA  NA    NA  [N]      99% [Y]
	#[Key = iDRAC.Embedded.1#SystemBoardMEMUsage]
	#System Board MEM Usage   Ok             0%      NA  NA    NA  [N]      99% [Y]
	#[Key = iDRAC.Embedded.1#SystemBoardSYSUsage]
	#System Board SYS Usage   Ok             2%      NA  NA    NA  [N]      99% [Y]
	elsif ( $sensor_type eq 'SYSTEM PERFORMANCE'
		&& m/(.*?) \s{2,40} (\S+) \s+ (\d+)% \s*
		    (NA|\d+%) \s* (NA|\d+%) \s*
		    (NA|\d+%) \s+ \[[YN]\] \s*
		    (NA|\d+%) \s+ \[[YN]\] /xi ) {
	    $sensor = $1;
	    $status = $2;
	    $reading = $3;
	    $verbose && print "> system performance $sensor $status $reading%\n";
	    if ( $status eq 'Critical' ) {
		push @crits, "$sensor $status $reading%";
		}
	    elsif ( $status ne 'Ok' ) {
		push @warns, "$sensor $status $reading%";
		}
	    }
	}

    push @oks, "$ncpus cpus";
    push @oks, "$ndimms dimms";
    push @oks, "$nfans fans";
    push @oks, "$nps power supplies";
    push @oks, "$ntemps temps";
    push @oks, "$nvoltages voltages";
    push @oks, "$ncurrents currents";

    }



sub get_racadm_cmd {
    my( $racadmcmd ) = @_;
    my( $cmd, $exp, $saw_pw_prompt, $line, $buf, $failed );

    $cmd = "/usr/local/bin/timeout -t 60 -- "
	. " /usr/local/bin/ssh -4akx -enone"
        . " -o StrictHostKeyChecking=no"
        . " -o BatchMode=no"
        . " -o ConnectTimeout=30"
        . " -o ServerAliveInterval=15"
	. " '$username\@$host'"
	. " racadm $racadmcmd";
    $verbose && print "cmd $cmd\n";

    if ( ! $verbose )  {
	# supposedly you can do $exp->log_stdout(0), but it doesn't work.
	$Expect::Log_Stdout = 0;
	}

    $exp = new Expect;
    $exp->raw_pty(1);
    unless ( $exp->spawn( $cmd ) ) {
        push @unknowns, "can't spawn ssh";
        }
    if ( $verbose > 1 ) {
        $exp->exp_internal(1);
        $exp->debug(1);
        }

    $failed = 0;
    $saw_pw_prompt = 0;
    $exp->expect( $timeout,
        [ qr/assword:/i, sub {
	    $saw_pw_prompt = 1;
	    } ],
	[ qr/permission.denied|could not resolve hostname [\w\d\.-]+/i, sub {
	    push @unknowns, $exp->match();
	    } ],
	);
    if ( $failed ) {
	$exp->hard_close();
	return;
	}
    if ( ! $saw_pw_prompt ) {
	push @crits, "didn't see password prompt";
	$exp->hard_close();
	ping_host( $host );
	return;
	}

    $exp->clear_accum();
    $exp->send( $password . "\r" );

    $failed = 0;
    $exp->expect( $timeout,
        [ qr/Password:/i, sub {
	    push @crits, "password rejected";
	    $failed = 1;
	    } ],
        [ qr/permission denied:/i, sub {
	    push @crits, "permission denied";
	    $failed = 1;
	    } ],
        [ qr/^[^\n]*\n/i, sub {
	    $buf .= $exp->match();
	    exp_continue;
	    } ],
	);
    if ( $failed ) {
	$exp->hard_close();
	return undef;
	}

    $exp->hard_close();

    return $buf;
    }






###################################





# return true if can ping host
sub ping {
    my( $host ) = @_;
    my( $cmd, $rc );

    $verbose && print "pinging host '$host'\n";

    $cmd = "/usr/local/sbin/fping -A -q -t 100 -p 100 '$host'";
    $verbose > 1 && print "+ $cmd\n";
    $rc = system( $cmd ) >> 8;
    if ( $rc ) {
	# failed
	$verbose && print "can't ping host '$host'\n";
	return 0;
	}
    else {
	# worked
	$verbose && print "successfully pinged host '$host'\n";
	return 1;
	}

    }





sub get_password {
    my( $file ) = @_;

    if ( ! open( fH, $file ) ) {
	push @unknowns, "can't open $file, $!";
	return;
	}
    my $pw = <fH>;
    close fH;
    chomp $pw;
    return $pw;
    }