#!/usr/local/bin/perl
#
# Check a netapp FAS3170
#
# $Header: /home/doke/work/nagios/RCS/check_netapp,v 1.38 2016/02/25 19:51:46 doke Exp $

use strict;
use warnings;
no warnings 'uninitialized';
no warnings 'redefine';
use Getopt::Long;
use Net::SNMP;
use Time::HiRes qw( usleep gettimeofday );
use Data::Dumper;


use vars qw( $community $timeout $host $verbose $help $mib2 $enterprises
    $netapp $crit_used $warn_used $crit_cpu $warn_cpu $prefered_maxmsgsize $maxrepetitions );

$community = 'public';
$crit_used = 90;   # percent
$warn_used = 80;   # percent
$crit_cpu = 100;   # percent
$warn_cpu = 98;    # percent
$timeout = 10;
$prefered_maxmsgsize = 1472;   # default
$host = '';
$verbose = 0;
$help = 0;

# try setting maxrepetitions to 1 to force get-next queries, instead of get-bulk
$maxrepetitions = 1;

$mib2 = '1.3.6.1.2.1';
$enterprises = '1.3.6.1.4.1';
$netapp = "$enterprises.789";

use vars qw( @ignore_vols @crits @warns @unknowns @oks @ignores $rc $sep );

sub usage {
    my( $rc ) = @_;
    print "Usage: $0 [options] -H <host> [-C <community>] 
    -H s  	hostname
    -C s  	snmp community [$community]
    -c n  	critical at n% used [$crit_used]
    -w n  	warn at n% used [$warn_used]
    --ccpu n    critical at n% cpu [$crit_cpu]
    --wcpu n    warn at n% cpu [$warn_cpu]
    -t n  	snmp timeout in seconds [$timeout]
    --ignore=vol  volume name to ignore (can use multiple times)
    -M n  	prefered snmp maxmsgsize to try first
    -v    	verbose
    -h    	help
";
    exit $rc;
    }

Getopt::Long::Configure ("bundling");
GetOptions(
    'H=s' => \$host,
    'C=s' => \$community,
    'c=i' => \$crit_used,
    'w=i' => \$warn_used,
    'ccpu=i' => \$crit_cpu,
    'wcpu=i' => \$warn_cpu,
    't=i' => \$timeout,
    'ignore=s' => \@ignore_vols,
    'M=i' => \$prefered_maxmsgsize,
    'v+' => \$verbose,
    'h' => \$help,
    );
&usage( 0 ) if ( $help );

&usage( 1 ) if ( ! $host );

if ( $crit_used < 1 || $crit_used > 100 ) { 
    warn "critical usage must be 1 to 100%\n";  
    usage( 1 );
    }
elsif ( $warn_used < 0 || $warn_used >= $crit_used ) { 
    warn "warning usage must be 0 to crit usage %\n";  
    usage( 1 );
    }

if ( $crit_cpu < 1 || $crit_cpu > 100 ) { 
    warn "critical cpu must be 1 to 100%\n";  
    usage( 1 );
    }
elsif ( $warn_cpu < 0 || $warn_cpu >= $crit_cpu ) { 
    warn "warning cpu must be 0 to crit cpu %\n";  
    usage( 1 );
    }


&check_netapp();

$rc = 0;
$sep = '';
if ( $#crits >= 0 ) {
    $rc = 2;
    print "CRITICAL ", join( ", ", @crits );
    $sep = '; ';
    }
if ( $#warns >= 0 ) {
    $rc = 1 if ( $rc == 0 );
    print $sep, "Warning ", join( ", ", @warns );
    $sep = '; ';
    }
if ( $#unknowns >= 0 ) {
    $rc = -1 if ( $rc == 0 );
    print $sep, "Unknown ", join( ", ", @unknowns );
    $sep = '; ';
    }
if ( $rc == 0 || $verbose ) {
    print $sep, "Ok ", join( ", ", @oks );
    $sep = '; ';
    }
if ( $#ignores >= 0 ) {
    print $sep, "Ignoring ", join( ", ", @ignores );
    }
print "\n";
exit $rc;


##################


sub check_netapp {
    my( $session, $error, $result, @oids, $row, $cmd, $pingout, $val,
	$table, $now, $vol, %lun_sums, %ignore_vols, $name, $status, $type,
	$nrows );
    

    # set up the ignore_vols lookups
    foreach $vol ( @ignore_vols ) { 
	if ( $vol =~ m%^/vol/([^/]+)%i ) { 
	    # /vol/vol_3170b_fc_install6/
	    $ignore_vols{ $1 } = 1;
	    }
	elsif ( $vol =~ m%^([^/]+)$%i ) { 
	    # vol_3170b_fc_install6
	    $ignore_vols{ $vol } = 1;
	    }
	}


    # open the snmp session
    $verbose && print "opening snmp session to $host\n";
    ( $session, $error ) = Net::SNMP->session(
        -version => 'snmpv2c',    # have to use v2c, lots of 64bit counters
        -hostname => $host,
        -community => $community,
        -timeout => $timeout,
        #-maxmsgsize => 2000,
        #-debug => 0x02
        );
    if ( ! defined( $session ) ) {
	$verbose && print "snmp setup error: $error\n";
        push @crits, "snmp setup error: $error";
        return;
        }
    $session->translate( [ '-octetstring' => 0, '-timeticks' => 0 ] );
    #$session->translate( '-all' => 0 );
    #$session->translate( '-unsigned' => 1 );

    $verbose > 1 && print "snmp session default maxmsgsize = ", 
	$session->max_msg_size(), "\n";


    # get the sysDescr and sysName
    # we'll need the sysDescr later to determine the ontap version, 
    # which controls which version of the mib to use
    my $sysDescr_oid = "$mib2.1.1.0";
    my $sysName_oid = "$mib2.1.5.0";
    @oids = ( $sysDescr_oid, $sysName_oid );
    $result = snmp_get( $session, "sysDescr and sysName", \@oids );
    if ( ! defined( $result ) ) {
	$verbose && print "snmp error ", $session->error(), "\n";
	push @crits, "couldn't get sysDescr " . $session->error();

	# we can't snmp to it, let's see if we can ping it
	$cmd = "ping $host 1 2>&1 |";
	$verbose && print "+ $cmd\n";
	if ( open( pH, $cmd ) ) {
	    $pingout = '';
	    while ( <pH> ) {
		$verbose && print ">$_";
		chomp;
		$pingout .= $_ . ', ';
		}
	    close pH;
	    $rc = $? >> 8;
	    if ( $rc ) {
		$pingout =~ s/, $//;
		push @crits, "ping failed: $pingout";
		}
	    else {
		push @crits, "ping worked, snmp may not be configured\n";
		}
	    }
	else {
	    push @crits, "unable to run ping: $!\n";
	    }
	return;
	}
    my $sysDescr = $result->{ $sysDescr_oid };
    my $sysName = $result->{ $sysName_oid };

    if ( $verbose ) {
	print "sysDescr = $sysDescr\n";
	print "sysName = $sysName\n";
	}
    $sysName =~ s/^([\w\d-]+)\..*/$1/;  # strip off the domain name for brevity
    push @oks, $sysName;

    if ( $sysDescr !~ m/NetApp/i ) {
	push @unknowns, "device doesn't look like a netapp: $sysDescr";
        return;
        }

    my $productVersion_oid = "$netapp.1.1.2.0";
    my $productModel_oid = "$netapp.1.1.5.0";
    my $product_fwv_oid = "$netapp.1.1.6.0";
    @oids = ( $productVersion_oid, $productModel_oid, $product_fwv_oid );
    $result = snmp_get( $session, "productVersion, productModel, and productFirmwareVersion", \@oids );
    if ( ! defined( $result ) ) {
	push @crits, "can't get productVersion, productModel, and productFirmwareVersion";
	return;
	}
    my $productVersion = $result->{ $productVersion_oid };
    my $productModel = $result->{ $productModel_oid };
    my $product_fwv = $result->{ $product_fwv_oid };
    if ( $verbose ) { 
	print "productVersion $productVersion\n";
	print "productModel $productModel\n";
	print "productFirmwareVersion $product_fwv\n";
	}
    if ( $productVersion !~ m/NetApp/i ) {
	push @unknowns, "device doesn't look like a netapp: $productVersion";
        return;
        }
    if ( $productModel ne 'FAS3170' && $productModel ne 'FAS3250' ) {
	push @unknowns, "This program only knows how to monitor NetApp FAS3170 or FAS3250, this productModel is '$productModel'";
        return;
        }
    my $version = '';
    if ( $productVersion =~ m/NetApp Release ([^:]+):/i ) { 
	$version = $1;
	}
    push @oks, "NetApp $productModel $version, fw $product_fwv";


    # looks like a netapp fas3170, good


    # global status
    my $miscGlobalStatus_oid = "$netapp.1.2.2.4.0";
    my $miscGlobalStatusMessage_oid = "$netapp.1.2.2.25.0";
    @oids = ( $miscGlobalStatus_oid, $miscGlobalStatusMessage_oid );
    $result = snmp_get( $session, "miscGlobalStatus", \@oids );
    if ( ! defined( $result ) ) {
	push @crits, "couldn't get miscGlobalStatus " . $session->error();
	}
    else {
	my $miscGlobalStatus = $result->{ $miscGlobalStatus_oid };
	my $miscGlobalStatusMessage = $result->{ $miscGlobalStatusMessage_oid };

	chomp $miscGlobalStatusMessage;
	$miscGlobalStatusMessage =~ s/\s*\n\s*/ /g;
	$miscGlobalStatusMessage =~ s/\s+$//;


	my @miscGlobalStatus_strings = ( 'unknown', 'other', 'unknown', 'ok', 'nonCritical',
	    'critical', 'nonRecoverable', 'unknown' );

	if ( $miscGlobalStatus == 5 || $miscGlobalStatus == 6 ) {
	    push @crits, "miscGlobalStatus $miscGlobalStatus_strings[ $miscGlobalStatus ], $miscGlobalStatusMessage";
	    }
	elsif ( $miscGlobalStatus == 4 ) {
	    push @warns, "miscGlobalStatus $miscGlobalStatus_strings[ $miscGlobalStatus ], $miscGlobalStatusMessage";
	    }
	elsif ( $miscGlobalStatus == 3 ) {
	    push @oks, "miscGlobalStatus $miscGlobalStatus_strings[ $miscGlobalStatus ]";
	    }
	else {
	    push @unknowns, "miscGlobalStatus $miscGlobalStatus_strings[ $miscGlobalStatus ], $miscGlobalStatusMessage";
	    }
	}




    # licenses
    $now = time();
    $table = snmp_table( $session, 'productLicenseTable', "$netapp.1.1.14", [ 2, 4, 6 ] );
    if ( $table ) {
	$nrows = scalar( @{$table} );
	foreach $row ( 1 .. $nrows - 1 ) {
	    next unless ( $table->[$row][2] );

	    # col 2 licenseService
	    # col 4 licenseIsExpired, 1 = false
	    # col 6 licenseExpirationTimeStamp, 1 = false
	    $verbose && print "    license $table->[$row][2] $table->[$row][4] $table->[$row][6]\n";
	    if ( $table->[$row][4] != 1 )  {
		push @crits, "license $table->[$row][2] is expired";
		}
	    if ( $table->[$row][6] > 0 ) {
		if ( $table->[$row][6] < $now )  {
		    push @crits, "license $table->[$row][2] is expired";
		    }
		elsif ( $table->[$row][6] < ( $now + 14 * 86400 ) )  {
		    push @warns, "license $table->[$row][2] is about to expire";
		    }
		}
	    }
	}





    # cpu
    $val = snmp_get_one( $session, 'cpuBusyTimePerCent', "$netapp.1.2.1.3.0" );
    if ( ! defined( $val ) ) {
	push @crits, "can't get cpuBusyTimePerCent";
	}
    else {
	$verbose && print "    cpuBusyTimePerCent $val\n";
	if ( $val > $crit_cpu ) {
	    push @crits, "cpu $val%";
	    }
	elsif ( $val > $warn_cpu ) {
	    push @warns, "cpu $val%";
	    }
	else {
	    push @oks, "cpu $val%";
	    }
	}



    # clustered failover status
    my $cfSettings_oid 			= "$netapp.1.2.3.1.0";
    my $cfState_oid 			= "$netapp.1.2.3.2.0";
    my $cfCannotTakeoverCause_oid 	= "$netapp.1.2.3.3.0";
    my $cfPartnerStatus_oid 		= "$netapp.1.2.3.4.0";
    my $cfPartnerLastStatusUpdate_oid 	= "$netapp.1.2.3.5.0";
    my $cfPartnerName_oid 		= "$netapp.1.2.3.6.0";
    my $cfPartnerSysid_oid 		= "$netapp.1.2.3.7.0";
    my $cfInterconnectStatus_oid 	= "$netapp.1.2.3.8.0";
    @oids = ( $cfSettings_oid,
	$cfState_oid,
	$cfCannotTakeoverCause_oid,
	$cfPartnerStatus_oid,
	$cfPartnerLastStatusUpdate_oid,
	$cfPartnerName_oid,
	#$cfPartnerSysid_oid,
	$cfInterconnectStatus_oid
	);
    $result = snmp_get( $session, "cfState", \@oids );
    if ( ! defined( $result ) ) {
	push @crits, "couldn't get cfState " . $session->error();
	}
    else {
	my $cfSettings = $result->{ $cfSettings_oid };
	my $cfState = $result->{ $cfState_oid };
	my $cfCannotTakeoverCause = $result->{ $cfCannotTakeoverCause_oid };
	my $cfPartnerStatus = $result->{ $cfPartnerStatus_oid };
	my $cfPartnerLastStatusUpdate = $result->{ $cfPartnerLastStatusUpdate_oid };
	my $cfPartnerName = $result->{ $cfPartnerName_oid };
	my $cfPartnerSysid = $result->{ $cfPartnerSysid_oid };
	my $cfInterconnectStatus = $result->{ $cfInterconnectStatus_oid };


	my @cfSettings_strings = ( 'unknown', 'notConfigured', 'enabled',
	    'disabled', 'takeoverByPartnerDisabled', 'thisNodeDead',
	    'unknown' );
	if ( $cfSettings == 5 ) { 
	    push @crits, "cf $cfSettings_strings[ $cfSettings ]";
	    }
	elsif ( $cfSettings == 3 || $cfSettings == 4 ) { 
	    push @warns, "cf $cfSettings_strings[ $cfSettings ]";
	    }
	elsif ( $cfSettings == 1 ) { 
	    push @ignores, "cf $cfSettings_strings[ $cfSettings ]";
	    }
	elsif ( $cfSettings == 2 ) { 
	    push @oks, "cf $cfSettings_strings[ $cfSettings ]";
	    }
	else { 
	    push @unknowns, "cf $cfSettings_strings[ $cfSettings ]";
	    }


	my @cfState_strings = ( 'unknown', 'dead', 'canTakeover',
	    'cannotTakeover', 'takeover', 'unknown' );
	if ( $cfState == 1 || $cfState == 3 || $cfState == 4 ) { 
	    push @crits, "cf $cfState_strings[ $cfState ]";
	    }
	elsif ( $cfState == 2 ) { 
	    push @oks, "cf $cfState_strings[ $cfState ]";
	    }
	else { 
	    push @unknowns, "cf $cfState_strings[ $cfState ]";
	    }


	my @cfCannotTakeoverCause_strings = ( 'unknown', 'ok', 'unknownReason',
	    'disabledByOperator', 'interconnectOffline', 'disabledByPartner', 'takeoverFailed', 'unknown' );
	if ( $cfCannotTakeoverCause == 2 || $cfCannotTakeoverCause == 4 || $cfCannotTakeoverCause == 5 || $cfCannotTakeoverCause == 6 ) { 
	    push @crits, "cf $cfCannotTakeoverCause_strings[ $cfCannotTakeoverCause ]";
	    }
	elsif ( $cfCannotTakeoverCause == 3 ) { 
	    push @warns, "cf $cfCannotTakeoverCause_strings[ $cfCannotTakeoverCause ]";
	    }
	elsif ( $cfCannotTakeoverCause == 1 ) { 
	    #push @oks, "cf $cfCannotTakeoverCause_strings[ $cfCannotTakeoverCause ]";
	    }
	else { 
	    push @unknowns, "cf $cfCannotTakeoverCause_strings[ $cfCannotTakeoverCause ]";
	    }


	my @cfPartnerStatus_strings = ( 'unknown', 'maybeDown', 'ok', 'dead', 'unknown' );
	if ( $cfPartnerStatus == 1 || $cfPartnerStatus == 3 ) { 
	    push @crits, "cf partner $cfPartnerName $cfPartnerStatus_strings[ $cfPartnerStatus ]";
	    }
	elsif ( $cfPartnerStatus == 2 ) { 
	    push @oks, "cf partner $cfPartnerName $cfPartnerStatus_strings[ $cfPartnerStatus ]";
	    }
	else { 
	    push @unknowns, "cf partner $cfPartnerName $cfPartnerStatus_strings[ $cfPartnerStatus ]";
	    }
	    
	$now = time();
	my $elapsed = $now - $cfPartnerLastStatusUpdate;
	if ( $elapsed > 300 ) { 
	    push @crits, "cf no update in $elapsed sec";
	    }
	elsif ( $elapsed > 60 ) { 
	    push @warns, "cf no update in $elapsed sec";
	    }
	elsif ( $elapsed < -30 ) { 
	    push @warns, "netapp time seems unsynchronized, cf update is in the future";
	    }
	else { 
	    push @oks, "cf partner update received $elapsed sec ago";
	    }

	my @cfInterconnectStatus_strings = ( 'unknown', 'notPresent',
	    'down', 'partialFailure', 'up', 'unknown' );
	if ( $cfInterconnectStatus == 1 || $cfInterconnectStatus == 2 || $cfInterconnectStatus == 3 ) { 
	    push @crits, "cf interconnect $cfInterconnectStatus_strings[ $cfInterconnectStatus ]";
	    }
	elsif ( $cfInterconnectStatus == 4 ) { 
	    push @oks, "cf interconnect $cfInterconnectStatus_strings[ $cfInterconnectStatus ]";
	    }
	else { 
	    push @unknowns, "cf interconnect $cfInterconnectStatus_strings[ $cfInterconnectStatus ]";
	    }
	}



    # environment
    my $envOverTemperature_oid 			= "$netapp.1.2.4.1.0";
    my $envFailedFanCount_oid 			= "$netapp.1.2.4.2.0";
    my $envFailedFanMessage_oid 		= "$netapp.1.2.4.3.0";
    my $envFailedPowerSupplyCount_oid 		= "$netapp.1.2.4.4.0";
    my $envFailedPowerSupplyMessage_oid 	= "$netapp.1.2.4.5.0";
    @oids = ( $envOverTemperature_oid,
	$envFailedFanCount_oid,
	$envFailedFanMessage_oid,
	$envFailedPowerSupplyCount_oid,
	$envFailedPowerSupplyMessage_oid
	);
    $result = snmp_get( $session, "environment", \@oids );
    if ( ! defined( $result ) ) {
	push @crits, "couldn't get environment " . $session->error();
	}
    else {
	my $envOverTemperature = $result->{ $envOverTemperature_oid };
	my $envFailedFanCount = $result->{ $envFailedFanCount_oid };
	my $envFailedFanMessage = $result->{ $envFailedFanMessage_oid };
	my $envFailedPowerSupplyCount = $result->{ $envFailedPowerSupplyCount_oid };
	my $envFailedPowerSupplyMessage = $result->{ $envFailedPowerSupplyMessage_oid };
	
	chomp $envFailedFanMessage;
	chomp $envFailedPowerSupplyMessage;

	if ( $envOverTemperature == 2 ) { 
	    push @crits, "over temperature";
	    }

	if ( $envFailedFanCount > 0 ) { 
	    push @warns, "$envFailedFanCount failed fans: $envFailedFanMessage";
	    }

	if ( $envFailedPowerSupplyCount > 0 ) { 
	    push @warns, "$envFailedPowerSupplyCount failed power supplys: $envFailedPowerSupplyMessage";
	    }
	}



    # nvram battery
    $val = snmp_get_one( $session, 'nvramBatteryStatus', "$netapp.1.2.5.1.0" );
    if ( ! defined( $val ) ) {
	push @crits, "can't get nvramBatteryStatus";
	}
    else {
	my @nvramBatteryStatus_strings = ( 'unknown', 'ok(1)',
	    'partiallyDischarged(2)', 'fullyDischarged(3)', 'notPresent(4)',
	    'nearEndOfLife(5)', 'atEndOfLife(6)', 'unknown(7)', 'overCharged(8)',
	    'fullyCharged(9)', 'unknown' );

	$verbose && print "    nvramBatteryStatus $nvramBatteryStatus_strings[ $val ]\n";
	if ( $val == 3 || $val == 4 || $val == 6 ) {
	    push @crits, "nvram battery $nvramBatteryStatus_strings[ $val ]";
	    }
	elsif ( $val == 2 || $val == 5 || $val == 8 ) {
	    push @warns, "nvram battery $nvramBatteryStatus_strings[ $val ]";
	    }
	elsif ( $val == 1 || $val == 9 ) {
	    push @oks, "nvram battery $nvramBatteryStatus_strings[ $val ]";
	    }
	else {
	    push @unknowns, "nvram battery $nvramBatteryStatus_strings[ $val ]";
	    }
	}



    # autosupport
    my $autosupportStatus_oid 			= "$netapp.1.2.7.1.0";
    my $autosupportStatusMessage_oid 		= "$netapp.1.2.7.2.0";
    @oids = ( $autosupportStatus_oid,
	$autosupportStatusMessage_oid
	);
    $result = snmp_get( $session, "autosupport", \@oids );
    if ( ! defined( $result ) ) {
	push @crits, "couldn't get autosupport " . $session->error();
	}
    else {
	my $autosupportStatus = $result->{ $autosupportStatus_oid };
	my $autosupportStatusMessage = $result->{ $autosupportStatusMessage_oid };

	chomp $autosupportStatusMessage;

	my @autosupportStatus_strings = ( 'unknown', 'ok(1)', 'smtpFailure(2)',
	    'postFailure(3)', 'smtpPostFailure(4)', 'unknown(5)', 'unknown' ); 
	$verbose && print "    autosupportStatus $autosupportStatus_strings[ $autosupportStatus ]\n";

	if ( $autosupportStatus != 1 ) { 
	    push @crits, "autosupport failed $autosupportStatus_strings[ $autosupportStatus ]: $autosupportStatusMessage";
	    }
	}

    

    # aggregates
    $table = snmp_table( $session, 'aggrTable', "$netapp.1.5.11", [ 2, 4, 5, 6 ] );
    if ( $table ) {
	$nrows = scalar( @{$table} );
	foreach $row ( 1 .. $nrows - 1 ) {
	    # col 2 is name
	    # col 4 is aggrOwningHost
	    # col 5 is aggrState
	    # col 6 is aggrStatus
	    $verbose && print "    $table->[$row][2] $table->[$row][4] $table->[$row][5] $table->[$row][6]\n";
	    # I don't know the whole list of things it can return here
	    if ( $table->[$row][5] ne 'online' || $table->[$row][6] =~ m/reconstructing/i )  {
		push @crits, "aggr $table->[$row][2] $table->[$row][5] $table->[$row][6]";
		}
	    }
	}




    # volumes
    # col 2 is name
    # col 4 is volOwningHost
    # col 5 is volState
    # col 6 is volStatus
    # this refuses to read, with a "too big" snmp error if the max message size is default 1472
    # works fine at 2000 or 3000
    $table = snmp_table( $session, 'volTable', "$netapp.1.5.8", [ 2, 4, 5, 6 ] );
    if ( $table ) {
	$nrows = scalar( @{$table} );
	foreach $row ( 1 .. $nrows - 1 ) {
	    $verbose && print "    $table->[$row][2] $table->[$row][4] $table->[$row][5] $table->[$row][6]\n";

	    if ( $ignore_vols{ $table->[$row][2] } )  {
		# ignore it
		}
	    # I don't know the whole list of things it can return here
	    elsif ( $table->[$row][5] =~ m/restricted/i )  {
		# ignore it
		}
	    elsif ( $table->[$row][6] =~ m/reconstructing/i )  {
		push @warns, "volume $table->[$row][2] $table->[$row][5] $table->[$row][6]";
		}
	    elsif ( $table->[$row][5] ne 'online' )  {
		push @crits, "volume $table->[$row][2] $table->[$row][5] $table->[$row][6]";
		}

	    $lun_sums{ $table->[$row][2] } = 0;
	    }
	}



    # qtrees
    # I think I need this to link luns and volumes?  maybe not 
    # indexed on 2 things: volFSID, and qtreeIndex
    # col 3 is volume name
    # col 5 is qtree name
    # col 7 is status
    $table = snmp_table_multi_index( $session, 'qtreeTable', "$netapp.1.5.10", [ 3, 5, 7 ], 2 );
    if ( $table ) {
	foreach $row ( sort keys %$table ) {
	    my @qtreeStatus_strings = ( 'unknown', 'normal', 'snapmirrored', 'snapvaulted' );
	    $verbose && print "    $table->{$row}[3] $table->{$row}[5] $qtreeStatus_strings[ $table->{$row}[7] ]\n";
	    }
	}



    # lunTable
    # col 2 is name
    # col 3 is comment
    # col 8 is lunQtreeName
    # col 17 is lunOnline
    # col 28 is lunSize64
    $table = snmp_table( $session, 'lunTable', "$netapp.1.17.15.2", [ 2, 3, 8, 17, 28 ] );
    if ( $table ) {
	$nrows = scalar( @{$table} );
	foreach $row ( 1 .. $nrows - 1 ) {
	    $verbose && print "    $table->[$row][2] $table->[$row][3] $table->[$row][8] $table->[$row][17] $table->[$row][28]\n";

	    $vol = $table->[$row][8];
	    #$verbose && print "    vol $vol\n";
	    if ( exists $lun_sums{ $vol } ) { 
		$lun_sums{ $vol } += $table->[$row][28] / 1024;
		}
	    else { 
		$vol = $table->[$row][2];
		$vol =~ s%/vol/([^/]+)/\S+%$1%i;
		#$verbose && print "    vol $vol\n";
		if ( exists $lun_sums{ $vol } ) { 
		    $lun_sums{ $vol } += $table->[$row][28] / 1024;
		    }
		}
	    }
	}

    if ( $verbose )  { 
	print "lun_sums:\n"; 
	foreach $vol ( sort keys %lun_sums ) { 
	    print "    $vol $lun_sums{ $vol }KB\n";
	    }
	}



    # dfTable
    # col 2 is name
    # col 6 is dfPerCentKBytesCapacity
    # col 9 is dfPerCentInodeCapacity
    # col 23 is dfType 
    # col 30 is df64UsedKBytes 
    $table = snmp_table( $session, 'dfTable', "$netapp.1.5.4", [ 2, 6, 9, 23, 30 ] );
    if ( $table ) {
	$nrows = scalar( @{$table} );
	foreach $row ( 1 .. $nrows - 1 ) {
	    my @dfType_strings = ( 'unknown', 'traditionalVolume',
		'flexibleVolume', 'aggregate', 'unknown' );

	    $verbose && print "    $table->[$row][2] $table->[$row][30]KB $table->[$row][6]%B $table->[$row][9]%i $dfType_strings[ $table->[$row][23] ]\n";

	    if ( $table->[$row][23] == 3 ) { 
		# don't bother warning about aggregates
		next;
		}

	    $vol = $table->[$row][2];
	    if ( $vol =~ m%/\.snapshot$% ) { 
		# ignore snapshots
		next;
		}

	    $vol =~ s%/vol/([^/]+)/\S*%$1%i;
	    if ( $ignore_vols{ $vol } )  {
		# ignore it
		next;
		}

	    if ( $table->[$row][6] > $warn_used || $table->[$row][9] > $warn_used )  { 
		my $matching = 0;
		if ( $lun_sums{ $vol } ) { 
		    $matching = $table->[$row][30] / $lun_sums{ $vol };
		    $verbose && printf( "        vol %s, lun size %ukB <=> kbytes used %ukB, %0.2f%%\n", 
			$vol, $lun_sums{ $vol }, $table->[$row][30], $matching );
		    }
		if ( 0.95 < $matching && $matching < 1.05 ) { 
		    $verbose && printf( "        sum of lun sizes %uKB roughly matchs kbytes used %uKB, %0.2f%%, ignoring\n", 
			$lun_sums{ $vol }, $table->[$row][30], $matching );
		    }
		elsif ( $table->[$row][6] > $crit_used )  {
		    push @crits, "$table->[$row][2] $table->[$row][6]% bytes used";
		    }
		elsif ( $table->[$row][9] > $crit_used )  {
		    push @crits, "$table->[$row][2] $table->[$row][9]% inodes used";
		    }
		elsif ( $table->[$row][6] > $warn_used )  {
		    push @warns, "$table->[$row][2] $table->[$row][6]% bytes used";
		    }
		elsif ( $table->[$row][9] > $warn_used )  {
		    push @warns, "$table->[$row][2] $table->[$row][9]% inodes used";
		    }
		}
	    else { 
		#push @oks, "$table->[$row][2] $table->[$row][6]% bytes used";
		#push @oks, "$table->[$row][2] $table->[$row][9]% inodes used";
		}
	    }
	}





    # diskSummary
    my $diskTotalCount_oid 			= "$netapp.1.6.4.1.0";
    my $diskActiveCount_oid 			= "$netapp.1.6.4.2.0";
    my $diskReconstructingCount_oid 		= "$netapp.1.6.4.3.0";
    my $diskReconstructingParityCount_oid 	= "$netapp.1.6.4.4.0";
    my $diskVerifyingParityCount_oid 		= "$netapp.1.6.4.5.0";
    my $diskScrubbingCount_oid 			= "$netapp.1.6.4.6.0";
    my $diskFailedCount_oid 			= "$netapp.1.6.4.7.0";
    my $diskSpareCount_oid 			= "$netapp.1.6.4.8.0";
    my $diskAddingSpareCount_oid 		= "$netapp.1.6.4.9.0";
    my $diskFailedMessage_oid 			= "$netapp.1.6.4.10.0";
    my $diskPrefailedCount_oid 			= "$netapp.1.6.4.11.0";
    @oids = ( $diskTotalCount_oid,
	$diskActiveCount_oid,
	$diskReconstructingCount_oid,
	$diskReconstructingParityCount_oid,
	$diskVerifyingParityCount_oid,
	$diskScrubbingCount_oid,
	$diskFailedCount_oid,
	$diskSpareCount_oid,
	$diskAddingSpareCount_oid,
	$diskFailedMessage_oid,
	$diskPrefailedCount_oid,
	);
    $result = snmp_get( $session, "diskSummary", \@oids );
    if ( ! defined( $result ) ) {
	push @crits, "couldn't get diskSummary " . $session->error();
	}
    else {
	my $diskTotalCount = $result->{ $diskTotalCount_oid };
	my $diskActiveCount = $result->{ $diskActiveCount_oid };
	my $diskReconstructingCount = $result->{ $diskReconstructingCount_oid };
	my $diskReconstructingParityCount = $result->{ $diskReconstructingParityCount_oid };
	my $diskVerifyingParityCount = $result->{ $diskVerifyingParityCount_oid };
	my $diskScrubbingCount = $result->{ $diskScrubbingCount_oid };
	my $diskFailedCount = $result->{ $diskFailedCount_oid };
	my $diskSpareCount = $result->{ $diskSpareCount_oid };
	my $diskAddingSpareCount = $result->{ $diskAddingSpareCount_oid };
	my $diskFailedMessage = $result->{ $diskFailedMessage_oid };
	my $diskPrefailedCount = $result->{ $diskPrefailedCount_oid };

	chomp $diskFailedMessage;

	if ( $diskReconstructingCount > 0 ) { 
	    push @warns, "reconstructing $diskReconstructingCount of $diskTotalCount disks";
	    }
	if ( $diskReconstructingParityCount > 0 ) { 
	    push @warns, "reconstructing parity on $diskReconstructingParityCount of $diskTotalCount disks";
	    }
	if ( $diskVerifyingParityCount > 0 ) { 
	    push @warns, "verifying parity on $diskVerifyingParityCount of $diskTotalCount disks";
	    }
	if ( $diskScrubbingCount > 0 ) { 
	    push @warns, "scrubbing $diskScrubbingCount of $diskTotalCount disks";
	    }
	if ( $diskFailedCount > 0 ) { 
	    push @warns, "failed $diskFailedCount of $diskTotalCount disks: $diskFailedMessage";
	    }

	if ( $diskSpareCount == 0 ) { 
	    push @crits, "no spare disks left";
	    }
	elsif ( $diskSpareCount < 2 ) { 
	    push @warns, "only 1 spare disk left";
	    }
	else { 
	    push @oks, "$diskSpareCount spares";
	    }

	if ( $diskAddingSpareCount > 0 ) { 
	    push @warns, "adding $diskAddingSpareCount spare disks";
	    }
	if ( $diskPrefailedCount > 0 ) { 
	    push @warns, "prefailed $diskPrefailedCount of $diskTotalCount disks";
	    }
	}





    # raidV table
    # indexed by raidVVol, raidVGroup, raidVIndex
    # this times out if the max message size is too big
    # fails to respond at 3000, works fine at 2000 
    my %raidVDiskTypes;
    my @raidVStatus_strings = ( 'unknown', 'active(1)',
	'reconstructionInProgress(2)',
	'parityReconstructionInProgress(3)',
	'parityVerificationInProgress(4)', 'scrubbingInProgress(5)',
	'failed(6)', 'unknown', 'unknown', 'prefailed(9)', 
	'offline(10)', 'unknown' );
    my $raidVStatus_string;
    $table = snmp_table_multi_index( $session, 'raidVTable', "$netapp.1.6.2", [ 2, 3, 31 ], 3 );
    if ( $table ) {
	foreach $row ( sort keys %$table ) {
	    # col 2 name
	    # col 3 status
	    # col 31 raidVDiskType
	    $name = $table->{$row}[2];
	    $status = $table->{$row}[3];
	    $type = $table->{$row}[31];
	    $raidVStatus_string = $raidVStatus_strings[ $status ];
	    $verbose && print "    $name $type $raidVStatus_string\n";
	    # I don't know the whole list of things it can return here
	    if ( $status == 6 || $status == 9 || $status == 10 )  {
		push @crits, "disk $name $type $raidVStatus_string";
		}
	    elsif ( $status == 2 || $status == 3 || $status == 4 
		    || $status == 5 )  {
		push @warns, "disk $name $type $raidVStatus_string";
		}
	    elsif ( $status == 1 )  {
		#push @oks, "disk $name $type $raidVStatus_strings";
		}
	    else {
		push @unknowns, "disk $name $type $raidVStatus_string";
		}

	    $raidVDiskTypes{ $type }++;
	    }
	}


    # spares table
    my %spareDiskTypes;
    my @spareStatus_strings = ( 'unknown', 'spare(1)',
	'addingspare(2)', 'bypassed(3)', 'unknown(4)', 'unknown(5)',
	'unknown(6)', 'unknown', 'unknown', 'unknown(9)',
	'offline(10)', 'unknown' );
    my $spareStatus_string;
    $table = snmp_table( $session, 'spareTable', "$netapp.1.6.3", [ 2, 3, 21 ] );
    if ( $table ) {
	$nrows = scalar( @{$table} );
	foreach $row ( 1 .. $nrows - 1 ) {
	    # col 2 name
	    # col 3 status
	    # col 21 spareDiskType
	    $name = $table->[$row][2];
	    $status = $table->[$row][3];
	    $type = $table->[$row][21];
	    $spareStatus_string = $spareStatus_strings[ $table->[$row][3] ];
	    $verbose && print "    $name $type $spareStatus_string\n";
	    if ( $status == 10 )  {
		push @crits, "$name $type $spareStatus_string";
		}
	    elsif ( $status == 2 || $status == 3 )  {
		push @warns, "$name $type $spareStatus_string";
		}
	    elsif ( $table->[$row][3] == 1 )  {
		#push @oks, "$name $type $spareStatus_string";
		}
	    else { 
		push @unknowns, "$name $type $spareStatus_string";
		}

	    $spareDiskTypes{ $type }++;
	    $raidVDiskTypes{ $type } = 0 if ( ! exists $raidVDiskTypes{ $type } );
	    }
	}


    # check there's at least one spare of each disk type 
    # todo: compare size and rpm too?
    # todo: compare spares pools?
    $verbose && print "spares comparison by type\n";
    foreach $type ( sort keys %raidVDiskTypes ) { 
	next if ( ! $type );
	$verbose && print "    $type: $raidVDiskTypes{ $type } in use, $spareDiskTypes{ $type } spares\n";
	if ( $spareDiskTypes{ $type } < 1 ) { 
	    push @crits, "no $type spare disks left";
	    }
	#elsif ( $spareDiskTypes{ $type } < 2 ) { 
	#    push @warns, "only one $type spare disk left";
	#    }
	}





    # todo: otherDiskTable

    # todo: raidPTable

    # todo: plexTable

    # todo: cifs


    if ( $sysDescr =~ m/NetApp Release 7.3.[34]/ ) { 

	$verbose && print "using mib 2.0.3\n";

	# enclTable table, in mib version 2.0.3, for ontap version 7.3.3 and 7.3.4
	# col 2 enclContactState
	# col 3 enclChannelShelfAddr   "1d.13.99", middle number shows on shelf LED
	# col 4 enclProductLogicalID    WWN, not very useful for nagios
	# col 9 enclProductSerialNo     space padded, not very useful for nagios
	# col 15 enclPowerSuppliesFailed
	# col 18 enclFansFailed
	# col 21 enclTempSensorsOverTempFail
	# col 22 enclTempSensorsOverTempWarn
	# col 23 enclTempSensorsUnderTempFail
	# col 24 enclTempSensorsUnderTempWarn
	# col 33 enclElectronicsFailed
	# col 36 enclVoltSensorsOverVoltFail
	# col 37 enclVoltSensorsOverVoltWarn
	# col 38 enclVoltSensorsUnderVoltFail
	# col 39 enclVoltSensorsUnderVoltWarn
	# col 47 enclCurSensorsOverCurFail
	# col 48 enclCurSensorsOverCurWarn
	$table = snmp_table( $session, 'enclTable', "$netapp.1.21.1.2", 
	    [ 2, 3, 15, 15, 18, 21, 22, 23, 24, 33, 36, 37, 38, 39, 47, 48 ] );
	if ( $table ) {
	    $nrows = scalar( @{$table} );
	    foreach $row ( 1 .. $nrows - 1 ) {

		my $shelfname = $table->[$row][3];
		$shelfname =~ s/([\da-f]+)\.(\d+)\.(\d+)/$2/i; 
		$shelfname =~ s/\s\s+/ /g; 
		$shelfname =~ s/\s+$//; 
		$shelfname = "shelf $shelfname";

		# enclContactState
		my @enclContactState_strings = ( 'unknown', 
		    'initializing(1)',
		    'transitioning(2)',
		    'active(3)',
		    'inactive(4)',
		    'reconfiguring(5)',
		    'nonexistent(6)',
		    'unknown' );

		#$table->[$row][9] =~ s/\s+$//;
		#$verbose && print "    $shelfname $table->[$row][3] $table->[$row][4] $table->[$row][9] $enclContactState_strings[ $table->[$row][2] ] $table->[$row][15] $table->[$row][18]\n";
		$verbose && print "    $shelfname $table->[$row][3] $enclContactState_strings[ $table->[$row][2] ] $table->[$row][15] $table->[$row][18]\n";

		if ( $table->[$row][2] == 1 || $table->[$row][2] == 4 || $table->[$row][2] == 6 )  {
		    push @crits, "$shelfname $enclContactState_strings[ $table->[$row][2] ]";
		    }
		elsif ( $table->[$row][2] == 2 || $table->[$row][2] == 5 )  {
		    push @warns, "$shelfname $enclContactState_strings[ $table->[$row][2] ]";
		    }
		elsif ( $table->[$row][2] == 3 )  {
		    #push @oks, "$shelfname $enclContactState_strings[ $table->[$row][2] ]";
		    }
		else { 
		    push @unknowns, "$shelfname $enclContactState_strings[ $table->[$row][2] ]";
		    }


		# enclPowerSuppliesFailed
		if ( $table->[$row][15] )  {
		    push @crits, "$shelfname has failed power supplies $table->[$row][16]";
		    }

		# enclFansFailed
		if ( $table->[$row][18] )  {
		    push @crits, "$shelfname has failed fans $table->[$row][20]";
		    }

		# enclTempSensorsOverTempFail
		if ( $table->[$row][21] )  {
		    push @crits, "$shelfname has over temperature alarms $table->[$row][21]";
		    }
		# enclTempSensorsOverTempWarn
		if ( $table->[$row][22] )  {
		    push @warns, "$shelfname has over temperature alarms $table->[$row][22]";
		    }
		# enclTempSensorsUnderTempFail
		if ( $table->[$row][23] )  {
		    push @crits, "$shelfname has under temperature alarms $table->[$row][23]";
		    }
		# enclTempSensorsUnderTempWarn
		if ( $table->[$row][24] )  {
		    push @warns, "$shelfname has under temperature alarms $table->[$row][24]";
		    }

		# enclElectronicsFailed
		if ( $table->[$row][33] )  {
		    push @crits, "$shelfname has failed electronics $table->[$row][33]";
		    }

		# enclVoltSensorsOverVoltFail
		if ( $table->[$row][36] )  {
		    push @crits, "$shelfname has over voltage alarms $table->[$row][36]";
		    }
		# enclVoltSensorsOverVoltWarn
		if ( $table->[$row][37] )  {
		    push @warns, "$shelfname has over voltage alarms $table->[$row][37]";
		    }
		# enclVoltSensorsUnderVoltFail
		if ( $table->[$row][38] )  {
		    push @crits, "$shelfname has under voltage alarms $table->[$row][38]";
		    }
		# enclVoltSensorsUnderVoltWarn
		if ( $table->[$row][39] )  {
		    push @warns, "$shelfname has under voltage alarms $table->[$row][39]";
		    }

		# enclCurSensorsOverCurFail
		if ( $table->[$row][47] )  {
		    push @crits, "$shelfname has over current alarms $table->[$row][47]";
		    }
		# enclCurSensorsOverCurWarn
		if ( $table->[$row][48] )  {
		    push @warns, "$shelfname has over current alarms $table->[$row][48]";
		    }

		}
	    }
	}

    elsif ( $sysDescr =~ m/NetApp Release 7.3.2/ ) { 

	$verbose && print "using mib 2.0.2\n";

	# enclTable table
	# enclTable table, in mib version 2.0.2, for ontap version 7.3.2 
	# col 2 enclContactState
	# col 3 enclChannelShelfAddr   "1d.13.99", middle number shows on shelf LED
	# col 4 enclProductLogicalID    WWN, not very useful for nagios
	# col 9 enclProductSerialNo     space padded, not very useful for nagios
	# col 16 enclPowerSuppliesFailed
	# col 20 enclFansFailed
	# col 23 enclTempSensorsOverTempFail
	# col 24 enclTempSensorsOverTempWarn
	# col 25 enclTempSensorsUnderTempFail
	# col 26 enclTempSensorsUnderTempWarn
	# col 37 enclElectronicsFailed
	# col 40 enclVoltSensorsOverVoltFail
	# col 41 enclVoltSensorsOverVoltWarn
	# col 42 enclVoltSensorsUnderVoltFail
	# col 43 enclVoltSensorsUnderVoltWarn
	# col 51 enclCurSensorsOverCurFail
	# col 52 enclCurSensorsOverCurWarn
	$table = snmp_table( $session, 'enclTable', "$netapp.1.21.1.2", 
	    [ 2, 3, 16, 20, 23, 24, 25, 26, 37, 40, 41, 42, 43, 51, 52 ] );
	if ( $table ) {
	    $nrows = scalar( @{$table} );
	    foreach $row ( 1 .. $nrows - 1 ) {

		my $shelfname = $table->[$row][3];
		$shelfname =~ s/([\da-f]+)\.(\d+)\.(\d+)/$2/i; 
		$shelfname =~ s/\s\s+/ /g; 
		$shelfname =~ s/\s+$//; 
		$shelfname = "shelf $shelfname";

		# enclContactState
		my @enclContactState_strings = ( 'unknown', 
		    'initializing(1)',
		    'transitioning(2)',
		    'active(3)',
		    'inactive(4)',
		    'reconfiguring(5)',
		    'nonexistent(6)',
		    'unknown' );

		#$table->[$row][9] =~ s/\s+$//;
		#$verbose && print "    $shelfname $table->[$row][3] $table->[$row][4] $table->[$row][9] $enclContactState_strings[ $table->[$row][2] ] $table->[$row][16] $table->[$row][20]\n";
		$verbose && print "    $shelfname $table->[$row][3] $enclContactState_strings[ $table->[$row][2] ] $table->[$row][16] $table->[$row][20]\n";

		if ( $table->[$row][2] == 1 || $table->[$row][2] == 4 || $table->[$row][2] == 6 )  {
		    push @crits, "$shelfname $enclContactState_strings[ $table->[$row][2] ]";
		    }
		elsif ( $table->[$row][2] == 2 || $table->[$row][2] == 5 )  {
		    push @warns, "$shelfname $enclContactState_strings[ $table->[$row][2] ]";
		    }
		elsif ( $table->[$row][2] == 3 )  {
		    #push @oks, "$shelfname $enclContactState_strings[ $table->[$row][2] ]";
		    }
		else { 
		    push @unknowns, "$shelfname $enclContactState_strings[ $table->[$row][2] ]";
		    }


		# enclPowerSuppliesFailed
		if ( $table->[$row][16] )  {
		    push @crits, "$shelfname has failed power supplies $table->[$row][16]";
		    }

		# enclFansFailed
		if ( $table->[$row][20] )  {
		    push @crits, "$shelfname has failed fans $table->[$row][20]";
		    }

		# enclTempSensorsOverTempFail
		if ( $table->[$row][23] )  {
		    push @crits, "$shelfname has over temperature alarms $table->[$row][23]";
		    }
		# enclTempSensorsOverTempWarn
		if ( $table->[$row][24] )  {
		    push @warns, "$shelfname has over temperature alarms $table->[$row][24]";
		    }
		# enclTempSensorsUnderTempFail
		if ( $table->[$row][25] )  {
		    push @crits, "$shelfname has under temperature alarms $table->[$row][25]";
		    }
		# enclTempSensorsUnderTempWarn
		if ( $table->[$row][26] )  {
		    push @warns, "$shelfname has under temperature alarms $table->[$row][26]";
		    }

		# enclElectronicsFailed
		if ( $table->[$row][37] )  {
		    push @crits, "$shelfname has failed electronics $table->[$row][37]";
		    }

		# enclVoltSensorsOverVoltFail
		if ( $table->[$row][40] )  {
		    push @crits, "$shelfname has over voltage alarms $table->[$row][40]";
		    }
		# enclVoltSensorsOverVoltWarn
		if ( $table->[$row][41] )  {
		    push @warns, "$shelfname has over voltage alarms $table->[$row][41]";
		    }
		# enclVoltSensorsUnderVoltFail
		if ( $table->[$row][42] )  {
		    push @crits, "$shelfname has under voltage alarms $table->[$row][42]";
		    }
		# enclVoltSensorsUnderVoltWarn
		if ( $table->[$row][43] )  {
		    push @warns, "$shelfname has under voltage alarms $table->[$row][43]";
		    }

		# enclCurSensorsOverCurFail
		if ( $table->[$row][51] )  {
		    push @crits, "$shelfname has over current alarms $table->[$row][51]";
		    }
		# enclCurSensorsOverCurWarn
		if ( $table->[$row][52] )  {
		    push @warns, "$shelfname has over current alarms $table->[$row][52]";
		    }

		}

	    }
	}

    elsif ( $sysDescr =~ m/NetApp Release 8/ ) { 
	$verbose && print "using mib 2.1.1\n";
	# FIXME, need mib 2.1.1
	}

    }








sub snmp_get {
    my( $session, $name, $oids ) = @_;
    my( $result, $oid, $val );

    $verbose && print "snmp_get $name\n";
    $result = $session->get_request( -varbindlist => $oids );
    if ( ! defined( $result ) ) {
        #warn "snmpget error: ", $session->error(), "\n";
        return undef;
        }

    foreach $oid ( @$oids ) {
	if ( ! exists $result->{ $oid } ) {
	    #warn "snmpget error: requested oid not in response\n";
	    $verbose && print "    $oid: no response\n";
	    $result->{ $oid } = undef;
	    }
	elsif ( $result->{ $oid } eq 'noSuchInstance' )  {
	    $verbose && print "    $oid: noSuchInstance\n";
	    $result->{ $oid } = undef;
	    }
	elsif ( $result->{ $oid } eq 'noSuchObject' )  {
	    $verbose && print "    $oid: noSuchObject\n";
	    $result->{ $oid } = undef;
	    }
	elsif ( $verbose ) {
	    $val = $result->{ $oid };
	    $val =~ tr/\040-\176//cd;
	    print "    $oid: $val\n";
	    }
	}

    return $result;
    }





sub snmp_get_one {
    my( $session, $name, $oid ) = @_;
    my( @oids, $result, $oid2, $val );

    $verbose && print "snmp_get $name\n";
    @oids = ( $oid );
    $result = $session->get_request( -varbindlist => \@oids );
    if ( ! defined( $result ) ) {
        #warn "snmpget error: ", $session->error(), "\n";
        return undef;
        }

    if ( ! exists $result->{ $oid } ) {
        #warn "snmpget error: requested oid not in response\n";
	$verbose && print "    $oid: no response\n";
        return undef;
        }
    elsif ( $result->{ $oid } eq 'noSuchInstance' )  {
	$verbose && print "    $oid: noSuchInstance\n";
        return undef;
        }
    elsif ( $result->{ $oid } eq 'noSuchObject' )  {
	$verbose && print "    $oid: noSuchObject\n";
        return undef;
        }

    if ( $verbose ) {
	$val = $result->{ $oid };
	$val =~ tr/\040-\176//cd;
	print "    $oid: $val\n";
	}
    return $result->{ $oid };
    }




sub snmp_walk {
    my( $session, $name, $baseoid ) = @_;
    my( $result );

    $verbose && print "walking $name\n";
    $result = $session->get_table( -baseoid => $baseoid, 
	-maxrepetitions => $maxrepetitions );
    #print "session error ", $session->error(), "\n";
    if ( ! defined( $result )
            && $session->error() !~ m/Requested table is empty/ ) {
        push @unknowns, sprintf( "error walking $name table on %s: %s",
            $session->hostname, $session->error() );
        return undef;
        }

    if ( $verbose > 1 ) {
	my( $oid, $val );
	foreach $oid ( sort keys %$result ) {
	    $val = $result->{ $oid };
	    print "$oid = $val\n";
	    }
	}

    return $result;
    }







# walk a table 
# With the optional columns parameter, just walk those selected columns out of it.  
# Return a pointer to a 2d array.  
sub snmp_table {
    my( $session, $name, $baseoid, $columns ) = @_;
    my( $result, $rows, $oid, $val, $col, $row, $data, @oids, $maxmsgsize );

    $verbose && print "walking $name table $baseoid @$columns\n";
    if ( defined $columns ) { 
	foreach $col ( @$columns ) { 
	    push @oids, "$baseoid.1.$col";
	    }
	}
    else { 
	push @oids, $baseoid;
	}

    $data = [];

    foreach $maxmsgsize ( $prefered_maxmsgsize, 1472, 1600, 1800, 2048, 4096, 9000 ) { 
	$session->max_msg_size( $maxmsgsize );
	$verbose > 1 && print "snmp get_entries tring maxmsgsize $maxmsgsize\n";
	$result = $session->get_entries( -columns => \@oids, 
	    -maxrepetitions => $maxrepetitions );
	if ( defined( $result ) ) {
	    last;
	    }
	elsif ( $session->error() =~ m/Requested table is empty/ ) {
	    # ok
	    return $data;
	    }
	elsif ( $session->error() =~ m/Message size exceeded buffer maxMsgSize/i ) {
	    # retry with different size
	    $verbose && print "snmp get_entries failed with maxmsgsize $maxmsgsize\n";
	    next;
	    }
	elsif ( $session->error() =~ m/The requested entries are empty or do not exist/i ) {
	    $verbose && print "snmp get_entries partially failed with error ", $session->error(), "\n";
	    return undef;
	    }
	else { 
	    $verbose && print "snmp get_entries failed with error ", $session->error(), "\n";
	    last;
	    }
	# else try a bigger maxmessage size
	}
    if ( ! defined( $result ) ) {
	push @crits, sprintf "error walking $name table on %s: %s",
	    $session->hostname, $session->error();
	return undef;
	}

    foreach $oid ( sort keys %$result ) {
	$val = $result->{ $oid };
	$verbose > 1 && print "$oid = $val\n";
	next if ( $val eq 'endOfMibView' );
	if ( $oid =~ m/.*\.(\d+)\.(\d+)$/ ) {
	    $col = $1; $row = $2;
	    $data->[$row][$col] = $val;
	    }
	}

    if ( scalar( @$data ) < 1 ) {
        push @unknowns, sprintf "no rows in $name table on %s: %s",
            $session->hostname, $session->error();
	return undef;
        }

    return $data;
    }









# walk a table where the "row" index is actually multiple levels in the oid 
# ie a 3 part index
sub snmp_table_multi_index {
    my( $session, $name, $baseoid, $columns, $nindexes ) = @_;
    my( $result, $rows, $oid, $val, $col, $row, $data, @oids, $maxmsgsize );

    $verbose && print "walking $name table $baseoid @$columns\n";

    if ( defined $columns ) { 
	foreach $col ( @$columns ) { 
	    push @oids, "$baseoid.1.$col";
	    }
	}
    else { 
	push @oids, $baseoid;
	}

    $data = {};
    foreach $maxmsgsize ( $prefered_maxmsgsize, 1472, 1600, 1800, 2048, 4096, 9000 ) { 
	$session->max_msg_size( $maxmsgsize );
	$verbose > 1 && print "snmp get_entries tring maxmsgsize $maxmsgsize\n";

	$result = $session->get_entries( -columns => \@oids, 
	    -maxrepetitions => $maxrepetitions );
	if ( defined( $result ) ) {
	    last;
	    }
	elsif ( $session->error() =~ m/Requested table is empty/ ) {
	    # ok
	    return $data;
	    }
	elsif ( $session->error() !~ m/Message size exceeded buffer maxMsgSize/ ) {
	    last;
	    }
	}
    if ( ! defined( $result ) ) {
	push @crits, sprintf "error walking $name table on %s: %s",
	    $session->hostname, $session->error();
	return undef;
	}

    foreach $oid ( sort keys %$result ) {
	$val = $result->{ $oid };
	$verbose > 1 && print "$oid = $val\n";
	next if ( $val eq 'endOfMibView' );
	if ( ( $nindexes == 3 && $oid =~ m/.*\.(\d+)\.(\d+\.\d+\.\d+)$/ ) 
		|| ( $nindexes == 2 && $oid =~ m/.*\.(\d+)\.(\d+\.\d+)$/ )
		|| ( $nindexes == 4 && $oid =~ m/.*\.(\d+)\.(\d+\.\d+\.\d+\.\d+)$/ ) ) {
	    $col = $1; $row = $2;
	    $data->{$row}[$col] = $val;
	    }
	}

    if ( scalar( keys %$data ) < 1 ) {
        push @unknowns, sprintf "no rows in $name table on %s: %s",
            $session->hostname, $session->error();
	return undef;
        }

    return $data;
    }