#!/usr/local/bin/perl # # nagios: -epn # Above line tells nagios not to use ePN. # Must be in first 10 lines of file. # # # Check a switch, cisco 3500xl, c3560, c3750, or juniper ex3300 # Also does switch-like parts of some routers: c6509, mx40, mx80, mx240, mx480 # # $Header: /home/doke/work/nagios/RCS/check_switch,v 1.131 2017/06/23 20:56:17 doke Exp $ # todo: check virtual chassis member serial numbers? # mysql # #create table sws ( # sw varchar(64) not null primary key, # sysUptime bigint unsigned not null default 0, # polled int not null default 1, # last_up int unsigned not null default 1 # ); #grant all on sws to nagios@localhost; # #create table swports ( # sw varchar(64) not null, # port varchar(24) not null, # ifAlias varchar(256) not null default '', # ifIndex int unsigned not null default 0, # dot1dBasePort int unsigned not null default 0, # trunk tinyint unsigned not null default 0, # ifAdminStatus tinyint unsigned not null default 0, # ifOperStatus tinyint unsigned not null default 0, # ifInErrors bigint unsigned not null default 0, # ifOutErrors bigint unsigned not null default 0, # ifInDiscards bigint unsigned not null default 0, # ifOutDiscards bigint unsigned not null default 0, # ifInErrors_rate float not null default 0.0, # ifOutErrors_rate float not null default 0.0, # ifInDiscards_rate float not null default 0.0, # ifOutDiscards_rate float not null default 0.0, # groupidx int unsigned not null default 0, # portidx int unsigned not null default 0, # poeStatus tinyint unsigned not null default 0, # polled int unsigned not null default 1, # last_up int unsigned not null default 1 # ); #create unique index swports_idx on swports ( sw, port ); #grant all on swports to nagios@localhost; # mysql -u nagios -p nagios # # show create table swports; # #alter table swports add column ifAlias varchar(256) not null default '' after port; #alter table swports add column ifIndex int unsigned not null default 0 after ifAlias; #alter table swports add column dot1dBasePort int not null default 0 after ifIndex; #alter table swports add column ifInErrors_rate float not null default 0.0 after ifOutDiscards; #alter table swports add column ifOutErrors_rate float not null default 0.0 after ifInErrors_rate; #alter table swports add column ifInDiscards_rate float not null default 0.0 after ifOutErrors_rate; #alter table swports add column ifOutDiscards_rate float not null default 0.0 after ifInDiscards_rate; #alter table swports add column groupidx int not null default 0 after ifOutDiscards_rate; #alter table swports add column portidx int not null default 0 after groupidx; #alter table swports add column poeStatus tinyint not null default 0 after portidx; #alter table swports add column last_up int unsigned not null default 1 after polled; # #alter table sws add column last_up int unsigned not null default 1 after polled; use strict; use warnings; no warnings 'uninitialized'; no warnings 'redefine'; use Getopt::Long; use Net::SNMP; use DBI; use Time::HiRes qw( usleep gettimeofday ); use XML::DOM; use Data::Dumper; use vars qw( $community $timeout $host $noop $verbose $help $mib2 $enterprises $cisco $juniper $warn_rate_errors $warn_rate_discards $prefered_maxmsgsize $maxrepetitions $dbname $dbuser $dbpasswd_file $maxrepetitions $retries $jnxBoxDescr @membernames $i @suppress_syslog_patterns $expected_vc_members $vc_members $swpasswd_file $one_interface @crits @warns @unknowns @oks @ignores $rc $sep @ports_to_expect_up %ports_to_expect_up $expect_all $port $session $dbh $sysDescr $sysUpTime $sysName %snmp %db %ifIndex_to_port $has_poe @ports_to_ignore %ports_to_ignore $vendor $model $check_trunks $ntrunks $force $min_poll_time %pethPsePortDetectionStatus_msgs $oper_down_window $version @perfdata $check_environment $highest_temp $highest_temp_msg $max_membername ); $community = 'public'; $warn_rate_errors = 10; # errors per minute $warn_rate_discards = 500; # discards per second $timeout = 2; $retries = 4; $prefered_maxmsgsize = 1472; # default $dbname = "nagios"; $dbuser = "nagios"; $dbpasswd_file = "/usr/local/nagios/etc/nagios.pw"; $swpasswd_file = "/usr/local/nagios/etc/nagios.pw"; $min_poll_time = 300; # don't repoll an interface if db data is fresher than this many seconds # How long an interface must be continuously operationally down, before we # just ignore it. Since the default "last_up" time is 1 second after epoch, # this should let us ignore interfaces that have never been up. # $oper_down_window = 86400 * 30; # try setting maxrepetitions to 1 to force get-next queries, instead of get-bulk #$maxrepetitions = 10; # juniper ex3300 with 11.4R2.14 hates this $maxrepetitions = 5; #$maxrepetitions = 1; # juniper ex3300 with 11.4R2.14 may not return entire table with this? # by default, we suppress certain syslog messages @suppress_syslog_patterns = ( 'Interface.*changed state', '0000.0000.fe0[01]', # Checkpoint firewall cluster protocol flapping messages 'Broadcast storm detected on .* packet filter action has been applied', 'Host .* in vlan .* is flapping between', 'Unblocking', 'decrypt: replay check failed', 'Authentication failure for SNMP', 'slot: \d+ - Buffers unavailable', 'rec\'d IPSEC packet has invalid spi for destaddr', 'duplex mismatch discovered on .* with SEP.* Port .* .half duplex', # cisco cdp voip phone 'duplex mismatch discovered on .*DESU', # desu networking errors ); $host = ''; $noop = 0; $verbose = 0; $help = 0; $expected_vc_members = 0; $vc_members = 0; $one_interface = ''; $has_poe = 0; $check_trunks = 0; $ntrunks = 0; $force = 0; $check_environment = 0; $highest_temp = 0; $highest_temp_msg = ''; $max_membername = 12; $mib2 = '1.3.6.1.2.1'; $enterprises = '1.3.6.1.4.1'; $cisco = "$enterprises.9"; $juniper = "$enterprises.2636"; %pethPsePortDetectionStatus_msgs = ( 0 => "unknown", 1 => "disabled", 2 => "searching", 3 => "deliveringPower", 4 => "fault", 5 => "test", 6 => "otherfault", 7 => "unknown", ); sub usage { my( $rc ) = @_; print "Usage: $0 [options] -H [-C ] Normally, doesn't check ports are up or down, just saves the data. In trunk mode (-T), criticals about any trunks that are administratively up but operationally down. In expect interface mode, criticals if listed ports are not up. In expect all mode, criticals about any ports that are administratively up but operationally down. In all modes, it saves the port data for later use by -o. -H s hostname -C s snmp community [$community] -m n number of members to expect --m0 s switch name for stack member 0 --m1 s switch name for stack member 1 ... --m12 s switch name for stack member 12 -o || --one_interface Lookup the last known state of One interface in the database. This doesn't do any snmp, it just peeks in the database. This implies -e, and is mutually exclusive with -expect_all. --env || --environment only check chassis environment, this requires ssh on junipers -T check trunks are up -e s port to expect to be up (repeatable) --expect_all expect all administratively enabled ports to be up, not just trunks -i s ignore this port (repeatable) overrides other options -w n warn at n errors/min [$warn_rate_errors] -d n warn at n discarded pkts/sec [$warn_rate_discards] -M n prefered snmp maxmsgsize to try first -t n snmp timeout in seconds [$timeout] -S p suppress syslog messages that match perlre pattern -f force getting fresh data in --one_interface mode -n noop, don't update database, for debugging -v verbose -h help "; exit $rc; } Getopt::Long::Configure ("bundling"); GetOptions( 'H=s' => \$host, 'C=s' => \$community, 'm=i' => \$expected_vc_members, 'm0=s' => \$membernames[ 0 ], 'm1=s' => \$membernames[ 1 ], 'm2=s' => \$membernames[ 2 ], 'm3=s' => \$membernames[ 3 ], 'm4=s' => \$membernames[ 4 ], 'm5=s' => \$membernames[ 5 ], 'm6=s' => \$membernames[ 6 ], 'm7=s' => \$membernames[ 7 ], 'm8=s' => \$membernames[ 8 ], 'm9=s' => \$membernames[ 9 ], 'm10=s' => \$membernames[ 10 ], 'm11=s' => \$membernames[ 11 ], 'm12=s' => \$membernames[ 12 ], # must match $max_membername 'S=s' => \@suppress_syslog_patterns, 'e=s' => \@ports_to_expect_up, 'expect_all' => \$expect_all, 'i=s' => \@ports_to_ignore, 'w=i' => \$warn_rate_errors, 'd=i' => \$warn_rate_discards, 't=i' => \$timeout, 'T' => \$check_trunks, 'M=i' => \$prefered_maxmsgsize, 'one_interface=s' => \$one_interface, 'o=s' => \$one_interface, 'environment' => \$check_environment, 'env' => \$check_environment, 'f' => \$force, 'n' => \$noop, 'v+' => \$verbose, 'h' => \$help, ); &usage( 0 ) if ( $help ); &usage( 1 ) if ( ! $host ); # fill in conventional member names # member 0 gets the target host name # other members that look like sw-ne37-115-20 get the id added to the last number # other members that look like something else get hostname-id if ( ! $membernames[ 0 ] ) { $membernames[ 0 ] = $host; } if ( $membernames[ 0 ] =~ m/^(.*)-(\d+)$/ ) { my $base = $1; my $nonce = $2; foreach $i ( 1 .. $max_membername ) { if ( ! $membernames[ $i ] ) { $membernames[ $i ] = sprintf "%s-%d", $base, $nonce + $i; } } } else { foreach $i ( 1 .. $max_membername ) { if ( ! $membernames[ $i ] ) { $membernames[ $i ] = sprintf "%s-%d", $membernames[ 0 ], $i; } } } if ( $one_interface ) { $one_interface =~ s/^\s*(.*?)\s*$/$1/; &open_db(); one_interface( $one_interface ); $dbh->disconnect(); } elsif ( $check_environment ) { &start_session(); if ( $session ) { &open_db(); if ( $dbh ) { &check_system(); if ( $sysDescr ) { usleep( 100000 ); # slow down to avoid overloading it's tiny brain. &check_alarms(); usleep( 100000 ); &check_env(); usleep( 250000 ); &check_env_long(); } } $dbh->disconnect(); } } else { &start_session(); if ( $session ) { &open_db(); if ( $dbh ) { &check_system(); if ( $sysDescr ) { usleep( 100000 ); # slow down to avoid overloading it's tiny brain. &check_alarms(); usleep( 100000 ); &check_env(); usleep( 250000 ); &check_power(); usleep( 250000 ); &check_poe(); usleep( 250000 ); # get historical interface data from database load_db_data( $host ); # hash ignore ports to remove any duplicates and for easier access # do this after loading the database, so cmd line overrides db monitor settings foreach $port ( @ports_to_ignore ) { $ports_to_ignore{ $port } = 1; delete $ports_to_expect_up{ $port }; } # hash expected ports to remove any duplicates and for easier access foreach $port ( @ports_to_expect_up ) { $ports_to_expect_up{ $port } = 1; delete $ports_to_ignore{ $port }; } &check_ports(); } } $dbh->disconnect(); } } if ( $highest_temp ) { push @oks, $highest_temp_msg; push @perfdata, "highest_temp=$highest_temp"; } $rc = 0; $sep = ''; if ( $#crits >= 0 ) { $rc = 2; map { s/\s+$// } @crits; print "CRITICAL ", join( ", ", @crits ); $sep = '; '; } if ( $#warns >= 0 ) { $rc = 1 if ( $rc == 0 ); map { s/\s+$// } @warns; print $sep, "Warning ", join( ", ", @warns ); $sep = '; '; } if ( $#unknowns >= 0 ) { $rc = 3 if ( $rc == 0 ); map { s/\s+$// } @unknowns; print $sep, "Unknown ", join( ", ", @unknowns ); $sep = '; '; } if ( $rc == 0 || $verbose ) { print $sep, "Ok ", join( ", ", @oks ); $sep = '; '; } if ( $#ignores >= 0 ) { map { s/\s+$// } @ignores; print $sep, "Ignoring ", join( ", ", @ignores ); } print( " | ", join( ", ", @perfdata ) ) if ( $#perfdata >= 0 ); print "\n"; exit $rc; ################## sub start_session { my( $error ); # open the snmp session $verbose && print "opening snmp session to $host\n"; ( $session, $error ) = Net::SNMP->session( -version => 'snmpv2c', -hostname => $host, -community => $community, -timeout => $timeout, -retries => $retries, #-maxmsgsize => 2000, #-debug => 0x02 ); if ( ! defined( $session ) ) { $verbose && print "snmp setup error: $error\n"; push @unknowns, "snmp setup error: $error"; return; } $session->translate( [ '-octetstring' => 0, '-timeticks' => 0 ] ); #$session->translate( '-all' => 0 ); #$session->translate( '-unsigned' => 1 ); $verbose > 1 && print "snmp session default maxmsgsize = ", $session->max_msg_size(), "\n"; } sub open_db { my( $dbpasswd ); $dbpasswd = &get_passwd( $dbpasswd_file ); $dbh = DBI->connect( "dbi:mysql:dbname=$dbname", $dbuser, $dbpasswd, { AutoCommit => 0, RaiseError => 0, PrintError => 1, } ); if ( ! $dbh ) { push @unknowns, "can't open database: " . $DBI::errstr; } } sub check_system { my( $result, @oids, $oid, $val, $now, $sysDescr_oid, $sysUpTime_oid, $sysName_oid, $sql, $sysUpTime_db, $polled_db, $last_up_db, $r, ); # get the sysDescr, sysName, and sysUpTime # we'll want the sysDescr later to determine the vendor and model # which controls which version of the mib to use $sysDescr_oid = "$mib2.1.1.0"; $sysUpTime_oid = "$mib2.1.3.0"; $sysName_oid = "$mib2.1.5.0"; @oids = ( $sysDescr_oid, $sysUpTime_oid, $sysName_oid ); $result = snmp_get( $session, "sysDescr, sysUptime and sysName", \@oids ); $now = time(); if ( ! defined( $result ) ) { $verbose && print "snmp error ", $session->error(), "\n"; push @unknowns, "couldn't get sysDescr " . $session->error(); return; } $sysDescr = $result->{ $sysDescr_oid }; $sysUpTime = $result->{ $sysUpTime_oid }; $sysName = $result->{ $sysName_oid }; if ( $verbose ) { print "sysDescr = $sysDescr\n"; print "sysUpTime = $sysUpTime\n"; print "sysName = $sysName\n"; } $sysName =~ s/^([\w\d-]+)\..*/$1/; # strip off the domain name for brevity push @oks, $sysName; if ( $sysDescr =~ m/Cisco .* Version (\d\S+), /is ) { $vendor = 'Cisco'; $version = $1; push @oks, "Cisco IOS $version"; } elsif ( $sysDescr =~ m/Palo Alto Networks (\S+)/is ) { $vendor = 'PaloAlto'; $model = $1; } else { # sysDescr is unreliable on juniper because you can change it # sysDescr might look like: Juniper Networks, Inc. ex3300-48p internet router, kernel JUNOS 11.4R2.14 #0: 2012-03-17 16:32:02 UTC # or not # so go for the juniper specific model number oid $jnxBoxDescr = snmp_get_one( $session, 'jnxBoxDescr', "$juniper.3.1.2.0" ); if ( $jnxBoxDescr || $sysDescr =~ m/Juniper Networks/i ) { # ok, it's a juniper something or other $vendor = 'Juniper'; # an srx240 in failover standby doesn't have a jnxBoxDescr in 12.1X47-D10.4 $jnxBoxDescr ||= $sysDescr; $model = $jnxBoxDescr; $model =~ s/.*Juniper (\S+) (?:Ethernet Switch|.*Router).*/$1/i; if ( $model ) { push @oks, "Juniper $model"; } #HOST-RESOURCES-MIB::hrSWInstalledName.2 = STRING: "JUNOS Base OS Software Suite [11.4R2.14]" $result = snmp_walk( $session, "hrSWInstalledName", "$mib2.25.6.3.1.2" ); foreach $oid ( keys %$result ) { $val = $result->{ $oid }; if ( $val =~ m/JUNOS .* \[(.*?)\]/i ) { $version = $1; push @oks, "JunOS $version"; last; } } } else { # let's see if it's a PaloAlto my $pa_model = snmp_get_one( $session, 'pa_model', "$enterprises.25461.2.1.2.2.1.0" ); if ( $pa_model ) { $vendor = 'PaloAlto'; $model = $pa_model; $version = snmp_get_one( $session, 'pa_version', "$enterprises.25461.2.1.2.1.1.0" ); } } } $vendor ||= 'unknown'; $verbose && print "vendor $vendor, model $model, version $version\n"; push @oks, "up " . &ticks_to_str( $sysUpTime ); push @perfdata, sprintf "uptime=%3.1f", $sysUpTime / 8640000; # convert ticks to days update_sysUpTime( $host, $sysUpTime, $now ); } # check_system sub update_sysUpTime { my( $host, $sysUpTime, $now ) = @_; my( $sql, $sysUpTime_db, $polled_db, $last_up_db, $r, $msg, @params ); $verbose && print "update_sysUpTime( $host, $sysUpTime, $now )\n"; $sql = "select sysUpTime, polled, last_up from sws where sw = ?"; ( $sysUpTime_db, $polled_db, $last_up_db ) = $dbh->selectrow_array( $sql, undef, ( $host ) ); # if uptime got smaller, then it either rebooted, or the counter rolled over # so ignore it if the prev_uptime was within an hour of rolling over # Remember, sysUpTime is in 100s of seconds, and rolls over every 497.1 days. if ( $sysUpTime < 720000 && $sysUpTime < $sysUpTime_db && $sysUpTime_db < 4294607296 ) { $msg = sprintf( "%s rebooted at %s, uptime changed from %s (%u) to %s (%u)", $host, scalar( localtime( time() - ( $sysUpTime / 100 ) ) ), ticks_to_str( $sysUpTime_db ), $sysUpTime_db, ticks_to_str( $sysUpTime ), $sysUpTime ); push @warns, $msg; } if ( defined $polled_db ) { $sql = "update sws set sysUpTime = ?, polled = ?, last_up = ? where sw = ?"; @params = ( $sysUpTime, $now, $now, $host ); $r = db_do( $sql, @params ); if ( $r != 1 ) { push @unknowns, "can't update database for $host: " . $DBI::errstr; } } else { $sql = "insert into sws ( sw, sysUpTime, polled, last_up ) values ( ?, ?, ?, ? )"; @params = ( $host, $sysUpTime, $now, $now ); $r = db_do( $sql, @params ); if ( $r != 1 ) { push @unknowns, "can't insert into database for $host: " . $DBI::errstr; } } } sub check_alarms { my( $table, $result, @oids ); if ( $vendor eq 'Cisco' ) { # CISCO-STACK-MIB::chassisMinorAlarm.0 = INTEGER: off(1) # CISCO-STACK-MIB::chassisMajorAlarm.0 = INTEGER: off(1) my $chassisMinorAlarm_oid = "$cisco.5.1.2.11.0"; my $chassisMajorAlarm_oid = "$cisco.5.1.2.12.0"; @oids = ( $chassisMinorAlarm_oid, $chassisMajorAlarm_oid ); $result = snmp_get( $session, "chassisGrp", \@oids ); if ( $result ) { if ( defined $result->{ $chassisMinorAlarm_oid } && $result->{ $chassisMinorAlarm_oid } != 1 ) { push @warns, "chassis minor alarm"; } if ( defined $result->{ $chassisMajorAlarm_oid } && $result->{ $chassisMajorAlarm_oid } != 1 ) { push @warns, "chassis major alarm"; } } # CISCO-SYSLOG-MIB::clogNotificationsSent.0 = Counter32: 9410 # CISCO-SYSLOG-MIB::clogNotificationsEnabled.0 = INTEGER: true(1) # CISCO-SYSLOG-MIB::clogMaxSeverity.0 = INTEGER: warning(5) # CISCO-SYSLOG-MIB::clogMsgIgnores.0 = Counter32: 13398 # CISCO-SYSLOG-MIB::clogMsgDrops.0 = Counter32: 0 # CISCO-SYSLOG-MIB::clogHistTableMaxLength.0 = INTEGER: 1 # CISCO-SYSLOG-MIB::clogHistMsgsFlushed.0 = Counter32: 9409 # CISCO-SYSLOG-MIB::clogHistFacility.9410 = STRING: "ILPOWER" # CISCO-SYSLOG-MIB::clogHistSeverity.9410 = INTEGER: error(4) # CISCO-SYSLOG-MIB::clogHistMsgName.9410 = STRING: "CONTROLLER_PORT_ERR" # CISCO-SYSLOG-MIB::clogHistMsgText.9410 = STRING: "Controller port error, Interface Gi0/45: Power Controller reports power Imax error detected" # CISCO-SYSLOG-MIB::clogHistTimestamp.9410 = Timeticks: (1027230690) 118 days, 21:25:06.90 # clogHistoryTable # 1 clogHistIndex Integer32, # 2 clogHistFacility DisplayString, # 3 clogHistSeverity SyslogSeverity, # emergency(1), # alert(2), # critical(3), # error(4), # warning(5), # notice(6), # info(7), # debug(8) # 4 clogHistMsgName DisplayString, # 5 clogHistMsgText DisplayString, # 6 clogHistTimestamp TimeStamp $table = snmp_table_multi_index( $session, 'clogHistoryTable', "$cisco.9.41.1.2.3", [ 3, 5, 6 ] ); #$verbose && print Dumper( $table ), "\n"; if ( $table ) { my( $key, $uptime, $severity, $msg ); foreach $key ( sort { $a <=> $b } keys %$table ) { $uptime = $table->{ $key }[ 6 ]; next if ( $sysUpTime < $uptime ); # sysUpTime wrapped since the log next if ( ( $sysUpTime - $uptime ) > 90000 ); # 15 minutes $severity = $table->{ $key }[ 3 ]; $msg = $table->{ $key }[ 5 ]; $verbose && print "syslog $severity '$msg'\n"; next if ( grep { $msg =~ m/$_/i } @suppress_syslog_patterns ); if ( $severity == 1 ) { # Cisco says it's an emergency push @crits, $msg; } elsif ( 2 <= $severity && $severity <= 5 ) { push @warns, $msg; } # else ignore it } } } elsif ( $jnxBoxDescr ) { @oids = ( '1.3.6.1.4.1.2636.3.4.2.2.2.0', # 'jnxYellowAlarmCount' '1.3.6.1.4.1.2636.3.4.2.3.2.0', # 'jnxRedAlarmCount' ); $result = snmp_get( $session, "jnxCraftAlarms", \@oids ); if ( $result ) { my( $red, $yellow ); $yellow = $result->{ '1.3.6.1.4.1.2636.3.4.2.2.2.0' }; $red = $result->{ '1.3.6.1.4.1.2636.3.4.2.3.2.0' }; $verbose && print "red $red, yellow $yellow\n"; if ( $red > 0 ) { push @warns, "$red red alarms"; } if ( $yellow > 0 ) { push @warns, "$yellow yellow alarms"; } if ( $red > 0 || $yellow > 0 ) { my( $output, $class, $msg, %critical_messages, %other_messages ); $output = ssh_cmd( "show system alarms; show chassis alarms" ); foreach $_ ( split( m/\n/, $output ) ) { if ( m/No alarms currently active/i ) { # good } elsif ( m/\d+ alarms currently active/i ) { # probably ought to check this against what snmp said } elsif ( m/^Alarm time/i ) { # column headers } elsif ( m/^\d{4}-\d{2}-\d{2} \s+ \d{1,2}:\d{2}:\d{2} \s+ \S+ \s+ (\S+) \s+ (\S.*)/x ) { $class = $1; $msg = $2; if ( $class =~ m/Critical/i ) { $critical_messages{ $msg } = 1; } else { $other_messages{ $msg } = 1; } } elsif ( m/WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED/i ) { push @unknowns, "REMOTE HOST IDENTIFICATION HAS CHANGED, fix $host in ~nagios/.ssh/known_hosts"; } } foreach $msg ( sort keys %critical_messages ) { # it hasn't killed the switch yet, so just warn next unless ( $msg ); push @warns, $msg; } foreach $msg ( sort keys %other_messages ) { next unless ( $msg ); push @warns, $msg; } } } } } # check_alarms sub check_env { my( @oids, $result, $oid, $val, %fanname, %tempname, %tempvalue, $id, $table, %stack, %serial_to_stack_id, $now, $nrows, %jnxContainersDescrs, %l1_to_stack_id, $model ); if ( $vendor eq 'Cisco' ) { #CISCO-STACK-MIB::chassisPs1Type.0 = INTEGER: other(1) #CISCO-STACK-MIB::chassisPs1Status.0 = INTEGER: ok(2) #CISCO-STACK-MIB::chassisPs1TestResult.0 = INTEGER: 0 #CISCO-STACK-MIB::chassisPs2Type.0 = INTEGER: externalPS(29) #CISCO-STACK-MIB::chassisPs2Status.0 = INTEGER: other(1) #CISCO-STACK-MIB::chassisPs2TestResult.0 = INTEGER: 0 #CISCO-STACK-MIB::chassisFanStatus.0 = INTEGER: ok(2) #CISCO-STACK-MIB::chassisFanTestResult.0 = INTEGER: 0 #CISCO-STACK-MIB::chassisMinorAlarm.0 = INTEGER: off(1) #CISCO-STACK-MIB::chassisMajorAlarm.0 = INTEGER: off(1) #CISCO-STACK-MIB::chassisTempAlarm.0 = INTEGER: off(1) #CISCO-STACK-MIB::chassisNumSlots.0 = INTEGER: 9 #CISCO-STACK-MIB::chassisSlotConfig.0 = INTEGER: 1 #CISCO-STACK-MIB::chassisModel.0 = STRING: "WS-C3560X-48PF-S" #CISCO-STACK-MIB::chassisSerialNumberString.0 = STRING: "FDO1521P06A" my $chassisPs1Status_oid = "$cisco.5.1.2.4.0"; my $chassisPs2Status_oid = "$cisco.5.1.2.7.0"; my $chassisFanStatus_oid = "$cisco.5.1.2.9.0"; #my $chassisMinorAlarm_oid = "$cisco.5.1.2.11.0"; #my $chassisMajorAlarm_oid = "$cisco.5.1.2.12.0"; my $chassisTempAlarm_oid = "$cisco.5.1.2.13.0"; @oids = ( $chassisPs1Status_oid, $chassisPs2Status_oid, $chassisFanStatus_oid, $chassisTempAlarm_oid ); $result = snmp_get( $session, "chassisGrp", \@oids ); if ( $result ) { if ( defined $result->{ $chassisPs1Status_oid } && $result->{ $chassisPs1Status_oid } != 2 ) { push @warns, "ps1 bad"; } if ( defined $result->{ $chassisPs2Status_oid } && $result->{ $chassisPs2Status_oid } != 2 && $result->{ $chassisPs2Status_oid } != 1 ) { push @warns, "ps2 bad"; } if ( defined $result->{ $chassisFanStatus_oid } && $result->{ $chassisFanStatus_oid } != 2 ) { push @warns, "chassis fan bad"; } if ( defined $result->{ $chassisTempAlarm_oid } && $result->{ $chassisTempAlarm_oid } != 1 ) { push @warns, "chassis temp alarm"; } } #CISCO-ENVMON-MIB::ciscoEnvMonTemperatureStatusDescr.1006 = STRING: "SW#1, Sensor#1, GREEN " #CISCO-ENVMON-MIB::ciscoEnvMonTemperatureStatusValue.1006 = Gauge32: 38 #CISCO-ENVMON-MIB::ciscoEnvMonTemperatureThreshold.1006 = INTEGER: 60 #CISCO-ENVMON-MIB::ciscoEnvMonTemperatureLastShutdown.1006 = INTEGER: 0 #CISCO-ENVMON-MIB::ciscoEnvMonTemperatureState.1006 = INTEGER: normal(1) $result = snmp_walk( $session, 'ciscoEnvMonTemperatureStatusTable', "$cisco.9.13.1.3" ); if ( $result ) { # sort oids, so names are defined before values are checked foreach $oid ( sort keys %$result ) { $val = $result->{ $oid }; $verbose > 1 && print "$oid = $val\n"; next if ( $val eq 'endOfMibView' ); if ( $oid =~ m/.*\.3\.1\.2\.(\d+)$/ ) { $tempname{ $1 } = $val; $tempname{ $1 } =~ s/\s+$//; } elsif ( $oid =~ m/.*\.3\.1\.3\.(\d+)$/ ) { $tempvalue{ $1 } = $val; } elsif ( $oid =~ m/.*\.3\.1\.6\.(\d+)$/ ) { # ciscoEnvMonTemperatureState # normal(1): # warning(2): # critical(3): # shutdown(4): # notPresent(5): # notFunctioning(6): if ( $val == 3 || $val == 4 || $val == 6 ) { # it hasn't killed the switch yet, so just warn push @warns, "temp '$tempname{ $1 }' bad $tempvalue{ $1 }"; } elsif ( $val == 2 ) { push @warns, "temp '$tempname{ $1 }' dubious $tempvalue{ $1 }"; } elsif ( $val == 1 ) { #push @oks, "temp '$tempname{ $1 }' ok $tempvalue{ $1 }"; } # else not present } } } # ciscoEnvMonFanStatusTable #CISCO-ENVMON-MIB::ciscoEnvMonFanStatusDescr.1058 = STRING: "Switch#1, Fan#1" #CISCO-ENVMON-MIB::ciscoEnvMonFanStatusDescr.1059 = STRING: "Switch#1, Fan#2" #CISCO-ENVMON-MIB::ciscoEnvMonFanState.1058 = INTEGER: normal(1) #CISCO-ENVMON-MIB::ciscoEnvMonFanState.1059 = INTEGER: normal(1) $result = snmp_walk( $session, 'ciscoEnvMonFanStatusTable', "$cisco.9.13.1.4" ); if ( $result ) { # sort oids, so names are defined before values are checked foreach $oid ( sort keys %$result ) { $val = $result->{ $oid }; $verbose > 1 && print "$oid = $val\n"; next if ( $val eq 'endOfMibView' ); if ( $oid =~ m/.*\.4\.1\.2\.(\d+)$/ ) { $fanname{ $1 } = $val; $fanname{ $1 } =~ s/\s+$//; } elsif ( $oid =~ m/.*\.4\.1\.3\.(\d+)$/ ) { # ciscoEnvMonFanState # normal(1): # warning(2): # critical(3): # shutdown(4): # notPresent(5): # notFunctioning(6): if ( $val == 3 || $val == 4 || $val == 6 ) { # it hasn't killed the switch yet, so just warn push @warns, "fan '$fanname{ $1 }' bad"; } elsif ( $val == 2 ) { push @warns, "fan '$fanname{ $1 }' dubious"; } elsif ( $val == 1 ) { #push @oks, "fan '$fanname{ $1 }' ok"; } # else not present } } } } # cisco elsif ( $vendor eq 'Juniper' ) { # the tests we did earlier say it's a juniper something if ( $jnxBoxDescr =~ m/Juniper (\S+) Ethernet Switch/i ) { $model = $1; } if ( $jnxBoxDescr =~ m/Juniper (\S+) Internet Backbone Router/i ) { $model = $1; } elsif ( $jnxBoxDescr =~ m/Juniper (\S+) (Mid|Back)plane Internet Backbone Router/i ) { $model = $1; } elsif ( $jnxBoxDescr =~ m/Juniper Virtual Chassis Ethernet Switch/i ) { # not useful, ignore } # todo: hrstorage stuff for logs filling up? # JUNIPER-HOSTRESOURCES-MIB::jnxHrStoragePercentUsed.1 = Gauge32: 53 # jnxVirtualChassisMemberTable # 1 jnxVirtualChassisMemberId INTEGER, # 2 jnxVirtualChassisMemberSerialnumber DisplayString, # 3 jnxVirtualChassisMemberRole 1 master, 2 backup, 3 linecard # 4 jnxVirtualChassisMemberMacAddBase MacAddress, # 5 jnxVirtualChassisMemberSWVersion DisplayString , # 6 jnxVirtualChassisMemberPriority INTEGER, # 7 jnxVirtualChassisMemberUptime INTEGER seems to be in seconds, not timeticks? # 8 jnxVirtualChassisMemberModel string # 9 jnxVirtualChassisMemberLocation string usually blank? # $table = snmp_table( $session, 'jnxVirtualChassisMemberTable', "$juniper.3.40.1.4.1.1", #[ 2, 3, 4, 5, 6, 7, 8 ] [ 2, 7, 8 ] ); #$verbose && print Dumper( $table ), "\n"; if ( $table && scalar( @$table ) > 0 ) { my( $now, $nrows, $id, $model ); $now = time(); $nrows = scalar( @$table ); foreach $id ( 0 .. $nrows - 1 ) { next if ( ! defined $table->[ $id ] ); $verbose && print "stack member id $id, ", join( ', ', @{$table->[ $id ]} ), "\n"; $stack{ $id }->{ 'serialnumber' } = $table->[ $id ][ 2 ]; #$stack{ $id }->{ 'role' } = $table->[ $id ][ 3 ]; #$stack{ $id }->{ 'macaddbase' } = $table->[ $id ][ 4 ]; #$stack{ $id }->{ 'swversion' } = $table->[ $id ][ 5 ]; #$stack{ $id }->{ 'priority' } = $table->[ $id ][ 6 ]; $stack{ $id }->{ 'uptime' } = $table->[ $id ][ 7 ]; $stack{ $id }->{ 'model' } = $table->[ $id ][ 8 ]; $serial_to_stack_id{ $table->[ $id ][ 2 ] } = $id; $verbose && print "serial_to_stack_id{ $table->[ $id ][ 2 ] } = $id\n"; # per-member uptimes if ( $membernames[ $id ] ne $host ) { update_sysUpTime( $membernames[ $id ], $table->[ $id ][ 7 ] * 100, $now ); } if ( ! $model ) { $model = $table->[ $id ][ 8 ]; } } } # jnxVirtualChassisMemberTable exists elsif ( $jnxBoxDescr =~ m/Virtual Chassis/i ) { # ssh out and do a 'show virtual-chassis status' my( $output, $id, $status ); $output = ssh_cmd( "show virtual-chassis status | display xml" ); foreach $_ ( split( m/\n/, $output ) ) { if ( m!(\d+)!i ) { $id = $1; } elsif ( m!(.+)!i ) { $status = $1; } elsif ( $status eq 'Prsnt' && m!(.+)!i ){ $stack{ $id }->{ 'serialnumber' } = $1; $serial_to_stack_id{ $1 } = $id; $verbose && print "serial_to_stack_id{ $1 } = $id\n"; } elsif ( $status eq 'Prsnt' && m!(.+)!i ) { $stack{ $id }->{ 'model' } = $1; if ( ! $model ) { $model = $table->[ $id ][ 8 ]; } } elsif ( m/WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED/i ) { push @unknowns, "REMOTE HOST IDENTIFICATION HAS CHANGED, fix $host in ~nagios/.ssh/known_hosts"; } } } if ( $model ) { push @oks, $model; } $vc_members = 0; my( $serialnumber ); foreach $serialnumber ( keys %serial_to_stack_id ) { if ( $serialnumber && $serialnumber ne 'BUILTIN' ) { $vc_members++; } } if ( $vc_members > 1 ) { push @oks, "stack of $vc_members"; } if ( $expected_vc_members > $vc_members ) { # we're missing a member, that means a sub-switch is down push @crits, "vc member count wrong, expected $expected_vc_members, found $vc_members"; } elsif ( $expected_vc_members && $expected_vc_members < $vc_members ) { # we found an extra, the database is wrong push @warns, "vc member count wrong, expected $expected_vc_members, found $vc_members"; } if ( $expected_vc_members > 1 || $vc_members > 1 ) { # jnxVirtualChassisPortTable # This is the admin and oper status for all the members vcp-* ports. # It doesn't them for a down member, but it does show the neighbor's vcp ports. # 3 jnxVirtualChassisPortAdminStatus 1 up, 2 down # 4 jnxVirtualChassisPortOperStatus 1 up, 2 down # $table = snmp_table_multi_index( $session, 'jnxVirtualChassisPortTable', "$juniper.3.40.1.4.1.2", [ 3, 4 ] ); #$verbose && print Dumper( $table ), "\n"; if ( $table ) { my( $index, $id, $portoid, $port, $admin, $oper ); foreach $index ( sort keys %$table ) { next if ( ! defined $table->{ $index }[ 3 ] ); ( $id, $portoid ) = split( m/\./, $index, 2 ); $port = pack( "C*", split( m/\./, $portoid ) ); $admin = $table->{ $index }[ 3 ]; $oper = $table->{ $index }[ 4 ]; $verbose && print "stack member id $id, member $membernames[ $id ], port $port, admin $admin, oper $oper\n"; if ( $admin == 1 && $oper == 2 ) { # admin up, oper down push @warns, "$membernames[ $id ] $port down"; } } } # jnxVirtualChassisPortTable exists } # jnxContainersDescr $result = snmp_walk( $session, 'jnxContainersDescr', "$juniper.3.1.6.1.6" ); if ( $result ) { foreach $oid ( keys %$result ) { $val = $result->{ $oid }; $verbose > 1 && print "$oid = $val\n"; next if ( $val eq 'endOfMibView' ); if ( $oid =~ m/\.6\.1\.6\.(\d+)$/ ) { $jnxContainersDescrs{ $1 } = $val; } } } # jnxContentsTable # this will have a 4 part index, ie 1.1.0.0 # the first part will be the type of container, jnxContainersIndex # the second part, L1, seems to be consistant for a chassis? # no idea what the third and 4th parts mean # 1 jnxContentsContainerIndex Integer32, # 2 jnxContentsL1Index Integer32, # 3 jnxContentsL2Index Integer32, # 4 jnxContentsL3Index Integer32, # 5 jnxContentsType OBJECT IDENTIFIER, # 6 jnxContentsDescr DisplayString, # 7 jnxContentsSerialNo DisplayString, # 8 jnxContentsRevision DisplayString, # 9 jnxContentsInstalled TimeStamp, # 10 jnxContentsPartNo DisplayString, # 11 jnxContentsChassisId JnxChassisId, # 12 jnxContentsChassisDescr DisplayString, # 13 jnxContentsChassisCleiCode DisplayString # $table = snmp_table_multi_index( $session, 'jnxContentsTable', "$juniper.3.1.8", [ 7 ] ); if ( $table ) { my( $content_id, $serialnumber, $stack_id, $container_id, $l1, $junk ); foreach $content_id ( sort keys %$table ) { ( $container_id, $l1, $junk, $junk ) = split( m/\./, $content_id ); $serialnumber = $table->{ $content_id }[ 7 ]; if ( $serialnumber && $serialnumber ne 'BUILTIN' && $container_id < 9 ) { # must not do this for a routing engine container, ie 9 # because they swap around oddly if the stack is running on # the backup master $stack_id = $serial_to_stack_id{ $serialnumber }; $l1_to_stack_id{ $l1 } = $stack_id; $verbose > 1 && print "l1_to_stack_id{ $l1 } = $stack_id\n"; } } } # jnxOperatingTable # 1 jnxOperatingContentsIndex Integer32, # 2 jnxOperatingL1Index Integer32, # 3 jnxOperatingL2Index Integer32, # 4 jnxOperatingL3Index Integer32, # 5 jnxOperatingDescr DisplayString, # 6 jnxOperatingState INTEGER, # unknown(1), # running(2), -- up and running, # -- as a active primary # ready(3), -- ready to run, not running yet # reset(4), -- held in reset, not ready yet # runningAtFullSpeed(5), # -- valid for fans only # down(6), -- down or off, for power supply # standby(7) -- running as a standby backup # 7 jnxOperatingTemp Gauge32, deg C # 8 jnxOperatingCPU Gauge32, percent # 9 jnxOperatingISR Gauge32, percent # 10 jnxOperatingDRAMSize Integer32, # 11 jnxOperatingBuffer Gauge32, percent # 12 jnxOperatingHeap Gauge32, # 13 jnxOperatingUpTime TimeInterval, # 14 jnxOperatingLastRestart TimeStamp, # 15 jnxOperatingMemory Integer32, # 16 jnxOperatingStateOrdered INTEGER, # 17 jnxOperatingChassisId JnxChassisId, # useless # 18 jnxOperatingChassisDescr DisplayString, # useless # 19 jnxOperatingRestartTime DateAndTime # 20 ? # 21 ? # 22 ? $table = snmp_table_multi_index( $session, 'jnxOperatingTable', "$juniper.3.1.13", [ 5, 6, 7, 8, 9, 11, 12 ] ); #$verbose && print Dumper( $table ), "\n"; if ( $table ) { my( $content_id, $stack_id, $name, $descr, $state, $temp, $cpu, $isr, $buffer, $heap, $container_id, $l1, $junk ); foreach $content_id ( sort keys %$table ) { ( $container_id, $l1, $junk, $junk ) = split( m/\./, $content_id ); $stack_id = $l1_to_stack_id{ $l1 }; if ( ! defined $stack_id ) { $stack_id = 0; # punt and assign to master } $name = $membernames[ $stack_id ]; $verbose > 1 && print "content_id $content_id, l1 $l1, stack_id $stack_id, name $name\n"; $descr = $table->{ $content_id }[ 5 ]; $state = $table->{ $content_id }[ 6 ]; $temp = $table->{ $content_id }[ 7 ]; $cpu = $table->{ $content_id }[ 8 ]; $isr = $table->{ $content_id }[ 9 ]; $buffer = $table->{ $content_id }[ 11 ]; $heap = $table->{ $content_id }[ 12 ]; if ( ! $descr ) { $descr = $jnxContainersDescrs{ $container_id }; } $verbose && print "$name $descr state $state\n"; if ( $state == 6 ) { if ( $descr eq 'CB 0' && $model =~ /\bMX[48]0\b/i ) { # MX80s and MX40s with 11.4R2.14 always report a # spurious "CB 0", control board 0, as down. # But it has a temperature. Wierd. # ignore it } else { push @warns, "$name $descr down"; } } elsif ( $state == 1 ) { # 11.4R2.14 seems to have trouble with ex3300 line card chassis fans? # shows them as "unknown" state if ( $jnxContainersDescrs{ $container_id } =~ m/fan/i ) { # ignore it } else { push @unknowns, "$name $descr unknown"; } } # http://kb.juniper.net/InfoCenter/index?page=content&id=KB22153 # # Junos temperature thresholds in SRX devices and the # actions taken when it exceeds the threshold # # "In general, the router increases the speed of the fans # when any component exceeds a temperature of 55 C. The # fans remain at the higher speed, until the temperature # decreases below the threshold. # # In this case, there is no service impact. However, if # the temperature exceeds 75 C, the router transmits a # warning and automatically shuts down." # http://kb.juniper.net/InfoCenter/index?page=content&id=KB10969&actp=LIST # # How to verify the component status, temperature, and # cooling system for the EX-series switch # # "To prevent the switch from overheating, do not operate it # in an area that exceeds the maximum recommended ambient # temperature of 104° F (40° C). To prevent airflow # restriction, allow at least 6 inches (15.2 cm) of # clearance around the ventilation openings." #doke@doke-ex3300> show chassis temperature-thresholds # Fan speed Yellow alarm Red alarm # (degrees C) (degrees C) (degrees C) #Item Normal High Normal Bad fan Normal Bad fan #FPC 0 CPU 60 70 80 70 95 85 #FPC 0 EX-PFE1 60 70 80 70 95 85 #FPC 0 GEPHY1 60 70 80 70 95 85 #FPC 0 Fan Exhaust 60 70 80 70 95 85 #FPC 0 SFP+ PHY 60 70 80 70 95 85 #FPC 0 Local Sensor 60 70 80 70 95 85 $verbose && $temp > 0 && print "$name $descr $temp C\n"; if ( $temp > 70 ) { # it hasn't killed the switch yet, so just warn push @warns, "$name $descr $temp C"; } if ( $temp > $highest_temp ) { $highest_temp = $temp; $highest_temp_msg = "$name $descr $temp C"; } $verbose && $cpu > 0 && print "$name $descr cpu $cpu %\n"; if ( $cpu > 98 ) { # it hasn't killed the switch yet, so just warn push @warns, "$name $descr cpu $cpu %"; } elsif ( $cpu > 90 ) { push @warns, "$name $descr cpu $cpu %"; } # interrupt service routine cpu utilization $verbose && $isr > 0 && print "$name $descr isr cpu $isr %\n"; if ( $isr > 95 ) { # it hasn't killed the switch yet, so just warn push @warns, "$name $descr isr cpu $isr %"; } elsif ( $isr > 90 ) { push @warns, "$name $descr isr cpu $isr %"; } # buffer memory utilization $verbose && $buffer > 0 && print "$name $descr buffer $buffer %\n"; if ( $buffer > 95 ) { # it hasn't killed the switch yet, so just warn push @warns, "$name $descr buffer $buffer %"; } elsif ( $buffer > 90 ) { push @warns, "$name $descr buffer $buffer %"; } # heap memory utilization $verbose && $heap > 0 && print "$name $descr heap $heap %\n"; if ( $heap > 95 ) { # it hasn't killed the switch yet, so just warn push @warns, "$name $descr heap $heap %"; } elsif ( $heap > 90 ) { push @warns, "$name $descr heap $heap %"; } } } # jnxOperatingTable exists } # it's a juniper elsif ( $vendor eq 'PaloAlto' ) { my( $table, $nrows, $i, %entities, $name, $value, $status, $units ); # entPhysicalTable.EntPhysicalEntry # 1 entPhysicalIndex PhysicalIndex, # 2 entPhysicalDescr SnmpAdminString, # 3 entPhysicalVendorType AutonomousType, # 4 entPhysicalContainedIn PhysicalIndexOrZero, # 5 entPhysicalClass PhysicalClass, # 6 entPhysicalParentRelPos Integer32, # 7 entPhysicalName SnmpAdminString, # 8 entPhysicalHardwareRev SnmpAdminString, # 9 entPhysicalFirmwareRev SnmpAdminString, # 10 entPhysicalSoftwareRev SnmpAdminString, # 11 entPhysicalSerialNum SnmpAdminString, # 12 entPhysicalMfgName SnmpAdminString, # 13 entPhysicalModelName SnmpAdminString, # 14 entPhysicalAlias SnmpAdminString, # 15 entPhysicalAssetID SnmpAdminString, # 16 entPhysicalIsFRU TruthValue, # 17 entPhysicalMfgDate DateAndTime, # 18 entPhysicalUris OCTET STRING $table = snmp_table( $session, 'entPhySensorTable', "$mib2.47.1.1.1", [ 2, 7 ] ); $nrows = scalar @$table; foreach $i ( 1 .. $nrows - 1 ) { $name = $table->[$i][2]; $name ||= $table->[$i][7]; $entities{ $i }{ name } = $name; } # entPhySensorTable.entPhySensorEntry # 1 entPhySensorType EntitySensorDataType, # 2 entPhySensorScale EntitySensorDataScale, # 3 entPhySensorPrecision EntitySensorPrecision, # 4 entPhySensorValue EntitySensorValue, # 5 entPhySensorOperStatus EntitySensorStatus, 1 == ok # 6 entPhySensorUnitsDisplay SnmpAdminString, # 7 entPhySensorValueTimeStamp TimeStamp, # 8 entPhySensorValueUpdateRate Unsigned32 # $table = snmp_table( $session, 'entPhySensorTable', "$mib2.99.1.1", [ 4, 5, 6 ] ); $nrows = scalar @$table; foreach $i ( 1 .. $nrows - 1 ) { $status = $table->[$i][5]; next unless ( defined $status ); $value = $table->[$i][4]; $units = $table->[$i][6]; $name = $entities{ $i }{ name }; $name ||= 'unknown entity'; # ok(1), # unavailable(2), # nonoperational(3) if ( $status == 1 ) { push @oks, "$name $value $units"; } else { push @warns, "$name not ok $value $units"; } } } else { push @unknowns, "unknown switch type"; } } # check_env # check the switch's chassis environment with "show chassis environment" # # ex3300 stack # doke@sw-nc41-219-20> show chassis environment #Class Item Status Measurement #Power FPC 0 Power Supply 0 OK # FPC 1 Power Supply 0 OK # FPC 2 Power Supply 0 OK #Temp FPC 0 CPU OK 52 degrees C / 125 degrees F # FPC 0 EX-PFE1 OK 62 degrees C / 143 degrees F # FPC 0 EX-PFE2 OK 55 degrees C / 131 degrees F # FPC 0 GEPHY1 OK 41 degrees C / 105 degrees F # FPC 0 Fan Exhaust OK 49 degrees C / 120 degrees F # FPC 0 SFP+ PHY OK 41 degrees C / 105 degrees F # FPC 0 Local Sensor OK 46 degrees C / 114 degrees F # FPC 1 CPU OK 52 degrees C / 125 degrees F #... #Fans FPC 0 Fan 1 OK Spinning at normal speed # FPC 0 Fan 2 OK Spinning at normal speed # FPC 1 Fan 1 OK Spinning at normal speed #... # # ex3000 stack with xml #doke@sw-nc41-219-20> show chassis environment | display xml # # # # FPC 0 Power Supply 0 # Power # OK # # # FPC 1 Power Supply 0 # Power # OK # # # FPC 2 Power Supply 0 # Power # OK # # # FPC 0 CPU # Temp # OK # 52 degrees C / 125 degrees F # # # FPC 0 EX-PFE1 # Temp # OK # 61 degrees C / 141 degrees F # # # FPC 0 EX-PFE2 # Temp # OK # 55 degrees C / 131 degrees F # # # FPC 0 GEPHY1 # Temp # OK # 41 degrees C / 105 degrees F # # # FPC 0 Fan Exhaust # Temp # OK # 48 degrees C / 118 degrees F # # # FPC 0 SFP+ PHY # Temp # OK # 40 degrees C / 104 degrees F # # # FPC 0 Local Sensor # Temp # OK # 46 degrees C / 114 degrees F # # # FPC 1 CPU # Temp # OK # 52 degrees C / 125 degrees F # # # FPC 1 EX-PFE1 # Temp # OK # 62 degrees C / 143 degrees F # # # FPC 1 EX-PFE2 # Temp # OK # 53 degrees C / 127 degrees F # # # FPC 1 GEPHY1 # Temp # OK # 40 degrees C / 104 degrees F # # # FPC 1 Fan Exhaust # Temp # OK # 48 degrees C / 118 degrees F # # # FPC 1 SFP+ PHY # Temp # OK # 40 degrees C / 104 degrees F # # # FPC 1 Local Sensor # Temp # OK # 45 degrees C / 113 degrees F # # # FPC 2 CPU # Temp # OK # 52 degrees C / 125 degrees F # # # FPC 2 EX-PFE1 # Temp # OK # 65 degrees C / 149 degrees F # # # FPC 2 EX-PFE2 # Temp # OK # 55 degrees C / 131 degrees F # # # FPC 2 GEPHY1 # Temp # OK # 41 degrees C / 105 degrees F # # # FPC 2 Fan Exhaust # Temp # OK # 49 degrees C / 120 degrees F # # # FPC 2 SFP+ PHY # Temp # OK # 41 degrees C / 105 degrees F # # # FPC 2 Local Sensor # Temp # OK # 45 degrees C / 113 degrees F # # # FPC 0 Fan 1 # Fans # OK # Spinning at normal speed # # # FPC 0 Fan 2 # Fans # OK # Spinning at normal speed # # # FPC 1 Fan 1 # Fans # OK # Spinning at normal speed # # # FPC 1 Fan 2 # Fans # OK # Spinning at normal speed # # # FPC 2 Fan 1 # Fans # OK # Spinning at normal speed # # # FPC 2 Fan 2 # Fans # OK # Spinning at normal speed # # # # {master:1} # # # # # ex4500 #doke@sw-nc41-219-8> show chassis environment #Class Item Status Measurement #Power FPC 0 Power Supply 0 OK # FPC 0 Power Supply 0 Airflow OK Front to back # FPC 0 Power Supply 1 OK # FPC 0 Power Supply 1 Airflow OK Front to back # FPC 0 Power Supply 2 Absent # FPC 0 Power Supply 3 Absent # FPC 0 Power Supply 4 Absent #Temp FPC 0 Rear Left PCB OK 56 degrees C / 132 degrees F # FPC 0 Rear Middle PCB OK 59 degrees C / 138 degrees F # FPC 0 Rear Right PCB OK 59 degrees C / 138 degrees F # FPC 0 PHY0 OK 47 degrees C / 116 degrees F # FPC 0 PHY6 OK 46 degrees C / 114 degrees F # FPC 0 PHY12 OK 45 degrees C / 113 degrees F # FPC 0 PHY19 OK 47 degrees C / 116 degrees F #Fans FPC 0 Fan 1 OK Spinning at normal speed # FPC 0 Fan 2 OK Spinning at normal speed # FPC 0 Fan 3 OK Spinning at normal speed # FPC 0 Fan 4 OK Spinning at normal speed # FPC 0 Fan 5 OK Spinning at normal speed # FPC 0 Fan-Tray Airflow OK Front to back # mx40 #doke@lerner-core1> show chassis environment #Class Item Status Measurement #Temp PEM 0 OK # PEM 1 OK # RE 0 Intake OK 33 degrees C / 91 degrees F # RE 0 Front Exhaust OK 36 degrees C / 96 degrees F # RE 0 Rear Exhaust OK 34 degrees C / 93 degrees F # Routing Engine OK 40 degrees C / 104 degrees F # Routing Engine CPU OK 50 degrees C / 122 degrees F # TFEB 0 QX 0 TSen OK 39 degrees C / 102 degrees F # TFEB 0 QX 0 Chip OK 48 degrees C / 118 degrees F # TFEB 0 LU 0 TSen OK 39 degrees C / 102 degrees F # TFEB 0 LU 0 Chip OK 53 degrees C / 127 degrees F # TFEB 0 MQ 0 TSen OK 39 degrees C / 102 degrees F # TFEB 0 MQ 0 Chip OK 46 degrees C / 114 degrees F # TFEB 0 TBB PFE TSen OK 42 degrees C / 107 degrees F # TFEB 0 TBB PFE Chip OK 55 degrees C / 131 degrees F # TFEB 0 TFEB PCIE TSen OK 40 degrees C / 104 degrees F # TFEB 0 TFEB PCIE Chip OK 54 degrees C / 129 degrees F #Fans Fan 1 OK Spinning at normal speed # Fan 2 OK Spinning at normal speed # Fan 3 OK Spinning at normal speed # Fan 4 OK Spinning at normal speed # Fan 5 OK Spinning at normal speed # # mx40 in xml #doke@lerner-core1> show chassis environment | display xml # # # # PEM 0 # Temp # OK # # # PEM 1 # Temp # OK # # # RE 0 Intake # Temp # OK # 33 degrees C / 91 degrees F # # # RE 0 Front Exhaust # Temp # OK # 36 degrees C / 96 degrees F # # # RE 0 Rear Exhaust # Temp # OK # 33 degrees C / 91 degrees F # # # Routing Engine # Temp # OK # 40 degrees C / 104 degrees F # # # Routing Engine CPU # Temp # OK # 50 degrees C / 122 degrees F # # # TFEB 0 QX 0 TSen # Temp # OK # 39 degrees C / 102 degrees F # # # TFEB 0 QX 0 Chip # Temp # OK # 48 degrees C / 118 degrees F # # # TFEB 0 LU 0 TSen # Temp # OK # 39 degrees C / 102 degrees F # # # TFEB 0 LU 0 Chip # Temp # OK # 53 degrees C / 127 degrees F # # # TFEB 0 MQ 0 TSen # Temp # OK # 39 degrees C / 102 degrees F # # # TFEB 0 MQ 0 Chip # Temp # OK # 46 degrees C / 114 degrees F # # # TFEB 0 TBB PFE TSen # Temp # OK # 42 degrees C / 107 degrees F # # # TFEB 0 TBB PFE Chip # Temp # OK # 55 degrees C / 131 degrees F # # # TFEB 0 TFEB PCIE TSen # Temp # OK # 40 degrees C / 104 degrees F # # # TFEB 0 TFEB PCIE Chip # Temp # OK # 54 degrees C / 129 degrees F # # # Fan 1 # Fans # OK # Spinning at normal speed # # # Fan 2 # Fans # OK # Spinning at normal speed # # # Fan 3 # Fans # OK # Spinning at normal speed # # # Fan 4 # Fans # OK # Spinning at normal speed # # # Fan 5 # Fans # OK # Spinning at normal speed # # # # # # # # mx240 with 10.4R12.4 doesn't always include the class # # # # PEM 0 # Temp # OK # 30 degrees C / 86 degrees F # # # PEM 1 # OK # 30 degrees C / 86 degrees F # # # PEM 2 # Absent # # # PEM 3 # Absent # # # Routing Engine 0 # OK # 27 degrees C / 80 degrees F # # # Routing Engine 0 CPU # OK # 22 degrees C / 71 degrees F # # # Routing Engine 1 # OK # 26 degrees C / 78 degrees F # # # Routing Engine 1 CPU # OK # 21 degrees C / 69 degrees F # # # CB 0 Intake # OK # 24 degrees C / 75 degrees F # # # CB 0 Exhaust A # OK # 24 degrees C / 75 degrees F # # # CB 0 Exhaust B # OK # 28 degrees C / 82 degrees F # # # CB 0 ACBC # OK # 25 degrees C / 77 degrees F # # # CB 0 SF A # OK # 36 degrees C / 96 degrees F # # # CB 0 SF B # OK # 28 degrees C / 82 degrees F # # # CB 1 Intake # OK # 24 degrees C / 75 degrees F # # # CB 1 Exhaust A # OK # 23 degrees C / 73 degrees F # # # CB 1 Exhaust B # OK # 27 degrees C / 80 degrees F # # # CB 1 ACBC # OK # 24 degrees C / 75 degrees F # # # CB 1 SF A # OK # 37 degrees C / 98 degrees F # # # CB 1 SF B # OK # 28 degrees C / 82 degrees F # # # FPC 1 Intake # OK # 33 degrees C / 91 degrees F # # # FPC 1 Exhaust A # OK # 29 degrees C / 84 degrees F # # # FPC 1 Exhaust B # OK # 37 degrees C / 98 degrees F # # # FPC 1 pfe0/I3 TSensor # OK # 33 degrees C / 91 degrees F # # # FPC 1 pfe0/I3 Chip # OK # 37 degrees C / 98 degrees F # # # FPC 1 pfe1/I3 TSensor # OK # 30 degrees C / 86 degrees F # # # FPC 1 pfe1/I3 Chip # OK # 32 degrees C / 89 degrees F # # # FPC 1 pfe0/XLR TSensor # OK # 36 degrees C / 96 degrees F # # # FPC 1 pfe0/XLR Chip # OK # 51 degrees C / 123 degrees F # # # FPC 1 pfe1/XLR TSensor # OK # 28 degrees C / 82 degrees F # # # FPC 1 pfe1/XLR Chip # OK # 46 degrees C / 114 degrees F # # # FPC 1 IA TSensor # OK # 37 degrees C / 98 degrees F # # # FPC 1 IA Chip # OK # 42 degrees C / 107 degrees F # # # FPC 1 Forbes 0 TSensor # OK # 22 degrees C / 71 degrees F # # # FPC 1 Forbes 0 Chip # OK # 40 degrees C / 104 degrees F # # # FPC 1 Forbes 1 TSensor # OK # 30 degrees C / 86 degrees F # # # FPC 1 Forbes 1 Chip # OK # 37 degrees C / 98 degrees F # # # FPC 2 Intake # OK # 25 degrees C / 77 degrees F # # # FPC 2 Exhaust A # OK # 35 degrees C / 95 degrees F # # # FPC 2 Exhaust B # OK # 43 degrees C / 109 degrees F # # # FPC 2 I3 0 TSensor # OK # 44 degrees C / 111 degrees F # # # FPC 2 I3 0 Chip # OK # 48 degrees C / 118 degrees F # # # FPC 2 I3 1 TSensor # OK # 41 degrees C / 105 degrees F # # # FPC 2 I3 1 Chip # OK # 41 degrees C / 105 degrees F # # # FPC 2 I3 2 TSensor # OK # 38 degrees C / 100 degrees F # # # FPC 2 I3 2 Chip # OK # 39 degrees C / 102 degrees F # # # FPC 2 I3 3 TSensor # OK # 33 degrees C / 91 degrees F # # # FPC 2 I3 3 Chip # OK # 34 degrees C / 93 degrees F # # # FPC 2 IA 0 TSensor # OK # 44 degrees C / 111 degrees F # # # FPC 2 IA 0 Chip # OK # 43 degrees C / 109 degrees F # # # FPC 2 IA 1 TSensor # OK # 39 degrees C / 102 degrees F # # # FPC 2 IA 1 Chip # OK # 42 degrees C / 107 degrees F # # # Front Fan # Fans # OK # Spinning at normal speed # # # Middle Fan # OK # Spinning at normal speed # # # Rear Fan # OK # Spinning at normal speed # # # # {master} # # # sub check_env_long { my( $output, $class, $name, %env_values, $status, $comment, $tempC, $nok, $nabsent, $nbad, $before, $paths, $path, $temperature, $junos_celsius ); return unless ( $jnxBoxDescr ); $before = gettimeofday(); $output = ssh_cmd( "show chassis environment | display xml" ); $verbose && printf "ssh took %0.6f s\n", gettimeofday() - $before; if ( $output =~ m/WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED/i ) { push @unknowns, "REMOTE HOST IDENTIFICATION HAS CHANGED, fix $host in ~nagios/.ssh/known_hosts"; return; } $output =~ s/^.*password://si; $before = gettimeofday(); eval { $paths = path_xml( $output ); }; $verbose && printf "parse_xml took %0.6f s\n", gettimeofday() - $before; $before = gettimeofday(); foreach $path ( keys %$paths ) { $name = $paths->{ $path }{ 'name' }; $class = $paths->{ $path }{ 'class' }; $status = $paths->{ $path }{ 'status' }; $comment = $paths->{ $path }{ 'comment' }; if ( ! $class ) { if ( $name =~ m/PEM/i ) { # power entrance module $class = 'Power'; } elsif ( $paths->{ $path }{ 'temperature' } ) { $class = 'Temp'; } elsif ( $comment =~ m/Spinning/i ) { $class = 'Fans'; } else { # no known class? $verbose && print "unknown class for $name, $status, $comment\n"; next; } } elsif ( $name =~ m/PEM/ && $paths->{ $path }{ 'temperature' } ) { # mx240 reports PEM 0 (only 0) as a temperature device # so enter it in both categories $env_values{ 'Temp' }{ $name }{ 'status' } = $status; $env_values{ 'Temp' }{ $name }{ 'comment' } = $paths->{ $path }{ 'temperature' }; $env_values{ 'Power' }{ $name }{ 'status' } = $status; $env_values{ 'Power' }{ $name }{ 'comment' } = $paths->{ $path }{ 'temperature' }; } $env_values{ $class }{ $name }{ 'status' } = $status; if ( $class eq "Temp" ) { $temperature = $paths->{ $path }{ 'temperature' }; $comment ||= $temperature; $junos_celsius = $paths->{ $path }{ 'junos:celsius' }; $env_values{ $class }{ $name }{ 'junos_celsius' } = $junos_celsius; $env_values{ $class }{ $name }{ 'temperature' } = $temperature; } else { $temperature = ''; $junos_celsius = ''; } $env_values{ $class }{ $name }{ 'comment' } = $comment; $verbose && print "$class, $name, $status, $comment, $junos_celsius, $temperature\n"; } $verbose && printf "parsing paths took %0.6f s\n", gettimeofday() - $before; # check the power items $nok = $nabsent = $nbad = 0; foreach $name ( sort keys %{$env_values{ 'Power' }} ) { $status = $env_values{ 'Power' }{ $name }{ 'status' }; $comment = $env_values{ 'Power' }{ $name }{ 'comment' }; $comment ||= ''; if ( $status eq 'OK' ) { next if ( $name =~ m/Airflow/ ); $nok++; } elsif ( $status eq 'Absent' ) { $nabsent++; } else { $nbad++; push @warns, "$name $status $comment"; } } if ( $nabsent > 0 && $nok < 2 ) { # we have less than two working power supplies in a multi-supply chassis push @warns, "no redundant power supply"; } push @oks, "$nok power-supplies ok"; # check the temperatures $nok = $nabsent = $nbad = 0; foreach $name ( sort keys %{$env_values{ 'Temp' }} ) { $status = $env_values{ 'Temp' }{ $name }{ 'status' }; $comment = $env_values{ 'Temp' }{ $name }{ 'temperature' }; $comment ||= $env_values{ 'Temp' }{ $name }{ 'comment' }; $comment ||= ''; $junos_celsius = $env_values{ 'Temp' }{ $name }{ 'junos_celsius' }; if ( $junos_celsius ) { $tempC = $junos_celsius; } elsif ( $comment =~ m/(\d+) degrees C/i ) { $tempC = $1; } elsif ( $comment =~ m/(\d+) degrees F/i ) { $tempC = ( $1 - 32 ) / 1.8; } else { $tempC = 0; } if ( $tempC > $highest_temp ) { $highest_temp = $tempC; $highest_temp_msg = "highest temp $tempC C"; } if ( $status eq 'OK' ) { $nok++; } elsif ( $status eq 'Absent' ) { # SRXs seem to always say Routing Engine CPU Temp is Absent # even when they have temperatures for it # ignore it } else { push @warns, "$name $status $tempC $comment"; $nbad++; } } push @oks, "$nok temps ok"; # check fans $nok = $nabsent = $nbad = 0; foreach $name ( sort keys %{$env_values{ 'Fans' }} ) { $status = $env_values{ 'Fans' }{ $name }{ 'status' }; $comment = $env_values{ 'Fans' }{ $name }{ 'comment' }; $comment ||= ''; if ( $status eq 'OK' ) { $nok++; $verbose && print "fan $name comment '$comment'\n"; if ( $comment =~ m/(?:full|high) speed/i ) { push @ignores, "$name $comment"; } } else { push @warns, "$name $status $comment"; $nbad++; } } push @oks, "$nok fans ok"; } # check_env_long sub path_xml { my( $xml ) = @_; my( $parser, $tree, $path, $tag, $paths, $tags, $names ); $verbose > 1 && print "path_xml( $xml )\n"; undef $paths; undef $tags; undef $names; $parser = new XML::DOM::Parser; $tree = $parser->parse( $xml ); #$verbose > 1 && print "tree: ", Dumper( $tree ), "\n\n"; $paths = path_xml_descend( $tree, 0, $paths, $tags, $names ); if ( $verbose > 1 ) { foreach $path ( keys %$paths ) { print "$path:\n"; foreach $tag ( sort keys %{$paths->{ $path }} ) { printf " %s = %s\n", $tag, $paths->{ $path }{ $tag }; } } } return $paths; } sub path_xml_descend { my( $node, $level, $paths, $tags, $names ) = @_; my( $tag, $type, $value, $path, $parenttag, @children, $child, $attrs, $attr, $n, $i ); $tag = $node->getNodeName(); $tags->[ $level ] = $tag; $names->[ $level ] ||= $tag; $type = $node->getNodeType(); if ( $level >= 2 ) { if ( $type == 3 || $type == 2 ) { # text or attribute node $value = $node->getNodeValue(); chomp $value; $value =~ s/\s+$//; $parenttag = $tags->[ $level - 1 ]; if ( $parenttag eq 'name' ) { $names->[ $level - 2 ] = $value; } $path = join( ' -> ', @$names[ 0 .. $level - 2 ] ); $paths->{ $path }{ $parenttag } = $value; $verbose > 1 && print "- $path, $parenttag = $value\n"; } elsif ( $type == 1 ) { $attrs = $node->getAttributes(); $n = $attrs->getLength(); if ( $n > 0 ) { $path = join( ' -> ', @$names[ 0 .. $level - 1 ] ); for ( $i = 0; $i < $n; $i++ ) { $attr = $attrs->item( $i ); $tag = $attr->getName(); $value = $attr->getValue(); $verbose > 1 && print "- $path, attr $n, tag $tag, value $value\n"; $paths->{ $path }{ $tag } = $value; } } } } @children = $node->getChildNodes(); foreach $child ( @children ) { $paths = path_xml_descend( $child, $level + 1, $paths, $tags, $names ); } return $paths; } # path_xml_descend sub check_power { my( @oids, $result, $count, $avgavail, $redundancy, $allocated, $redundant_available ); if ( $jnxBoxDescr ) { # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuAvailableDeviceCount.0 = INTEGER: 6 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuAvailableAveragePowerSupply.0 = INTEGER: 10400 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuAvailableMaxPowerSupply.0 = INTEGER: 10400 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuRedundancy.0 = INTEGER: nPlusOneRedundancy(2) # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuChassisPowerReserved.0 = INTEGER: 1600 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuChassisPowerAllocated.0 = INTEGER: 3550 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuRedundantPowerAvailable.0 = INTEGER: 4850 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuTotalPowerAvailable.0 = INTEGER: 6850 @oids = ( "$juniper.3.58.1.2.1.1.0", # 'jnxPsuAvailableDeviceCount' "$juniper.3.58.1.2.1.2.0", # 'jnxPsuAvailableAveragePowerSupply' #"$juniper.3.58.1.2.1.3.0", # 'jnxPsuAvailableMaxPowerSupply' "$juniper.3.58.1.2.1.4.0", # 'jnxPsuRedundancy' #"$juniper.3.58.1.2.1.5.0", # 'jnxPsuChassisPowerReserved' "$juniper.3.58.1.2.1.6.0", # 'jnxPsuChassisPowerAllocated' "$juniper.3.58.1.2.1.7.0", # 'jnxPsuRedundantPowerAvailable' #"$juniper.3.58.1.2.1.8.0", # 'jnxPsuTotalPowerAvailable' #"$juniper.3.58.1.2.1.9.0", # 'jnxPsuChassisPowerConsumed' ); $result = snmp_get( $session, "jnxPsuScalars", \@oids ); if ( $result ) { $count = $result->{ "$juniper.3.58.1.2.1.1.0" }; $avgavail = $result->{ "$juniper.3.58.1.2.1.2.0" }; #$maxavail = $result->{ "$juniper.3.58.1.2.1.3.0" }; $redundancy = $result->{ "$juniper.3.58.1.2.1.4.0" }; #$reserved = $result->{ "$juniper.3.58.1.2.1.5.0" }; $allocated = $result->{ "$juniper.3.58.1.2.1.6.0" }; $redundant_available = $result->{ "$juniper.3.58.1.2.1.7.0" }; #$total_available = $result->{ "$juniper.3.58.1.2.1.8.0" }; #$consumed = $result->{ "$juniper.3.58.1.2.1.9.0" }; if ( $count ) { push @oks, "$count PSUs, $allocated W allocated, $redundant_available W available"; if ( $redundancy == 3 ) { push @warns, "no power supply redundancy"; } if ( $redundant_available < 1 ) { push @warns, "no redundant power left"; } } # todo later, maybe # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuAvgPower.2.1.0.0 = INTEGER: 2000 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuAvgPower.2.2.0.0 = INTEGER: 2000 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuAvgPower.2.3.0.0 = INTEGER: 1200 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuAvgPower.2.4.0.0 = INTEGER: 2000 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuAvgPower.2.5.0.0 = INTEGER: 2000 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuAvgPower.2.6.0.0 = INTEGER: 1200 # # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuMaxPower.2.1.0.0 = INTEGER: 2000 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuMode.2.1.0.0 = INTEGER: single(1) # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuOutletCount.2.1.0.0 = INTEGER: 1 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuThermalValue.2.1.0.0 = INTEGER: 25 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuHumidityValue.2.1.0.0 = INTEGER: 0 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuOutletName.2.1.0.0 = STRING: PSU 0 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuOutletDescription.2.1.0.0 = STRING: EX8200-AC2K # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuOutletAvgPower.2.1.0.0 = INTEGER: 2000 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuOutletMaxPower.2.1.0.0 = INTEGER: 2000 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuOutletCurrent.2.1.0.0 = INTEGER: 3 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuOutletStatus.2.1.0.0 = INTEGER: on(1) # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuFpcPowerPriority.7.2.0.0 = INTEGER: 7 # JUNIPER-POWER-SUPPLY-UNIT-MIB::jnxPsuFpcPowerAllocated.7.2.0.0 = INTEGER: 550 # # #'1.3.6.1.4.1.2636.3.58.1.2.2' => 'jnxPsuTable', #'1.3.6.1.4.1.2636.3.58.1.2.2.1' => 'jnxPsuEntry', #'1.3.6.1.4.1.2636.3.58.1.2.2.1.1' => 'jnxPsuAvgPower', #'1.3.6.1.4.1.2636.3.58.1.2.2.1.2' => 'jnxPsuMaxPower', #'1.3.6.1.4.1.2636.3.58.1.2.2.1.3' => 'jnxPsuMode', #'1.3.6.1.4.1.2636.3.58.1.2.2.1.4' => 'jnxPsuOutletCount', #'1.3.6.1.4.1.2636.3.58.1.2.3' => 'jnxPsuEnvironmentTable', #'1.3.6.1.4.1.2636.3.58.1.2.3.1' => 'jnxPsuEnvironmentEntry', #'1.3.6.1.4.1.2636.3.58.1.2.3.1.1' => 'jnxPsuThermalValue', #'1.3.6.1.4.1.2636.3.58.1.2.3.1.2' => 'jnxPsuHumidityValue', #'1.3.6.1.4.1.2636.3.58.1.2.4' => 'jnxPsuOutletTable', #'1.3.6.1.4.1.2636.3.58.1.2.4.1' => 'jnxPsuOutletEntry', #'1.3.6.1.4.1.2636.3.58.1.2.4.1.1' => 'jnxPsuOutletName', #'1.3.6.1.4.1.2636.3.58.1.2.4.1.2' => 'jnxPsuOutletDescription', #'1.3.6.1.4.1.2636.3.58.1.2.4.1.3' => 'jnxPsuOutletAvgPower', #'1.3.6.1.4.1.2636.3.58.1.2.4.1.4' => 'jnxPsuOutletMaxPower', #'1.3.6.1.4.1.2636.3.58.1.2.4.1.5' => 'jnxPsuOutletCurrent', #'1.3.6.1.4.1.2636.3.58.1.2.4.1.8' => 'jnxPsuOutletStatus', #'1.3.6.1.4.1.2636.3.58.1.2.4.1.9' => 'jnxPsuOutletVoltage', #'1.3.6.1.4.1.2636.3.58.1.2.4.1.10' => 'jnxPsuOutletPowerFactorValue', #'1.3.6.1.4.1.2636.3.58.1.2.4.1.11' => 'jnxPsuOutletPowerConsumed', #'1.3.6.1.4.1.2636.3.58.1.2.5' => 'jnxPsuFpcPowerTable', #'1.3.6.1.4.1.2636.3.58.1.2.5.1' => 'jnxPsuFpcPowerEntry', #'1.3.6.1.4.1.2636.3.58.1.2.5.1.1' => 'jnxPsuFpcPowerPriority', #'1.3.6.1.4.1.2636.3.58.1.2.5.1.2' => 'jnxPsuFpcPowerAllocated', } } } # check_power # global poe status sub check_poe { my( $table, $nrows, $i, $id, $idx, $group, $port, $name, $avail, $consumed, $total_avail, $total_consumed, $util, $oid, $result, $val ); # mib-2.powerEthernetMIB.pethObjects.pethMainPseObjects.pethMainPseTable.pethMainPseEntry # 1 pethMainPseGroupIndex INTEGER, # 2 pethMainPsePower Gauge, in watts # 3 pethMainPseOperStatus INTEGER, # 4 pethMainPseConsumptionPower Gauge, supposed to be in milliwats # 5 pethMainPseUsageThreshold INTEGER # $table = snmp_table( $session, 'pethMainPseTable', "$mib2.105.1.3.1", [ 2, 4 ] ); return if ( ! $table ); # doesn't have poe, or snmp sub-daemon died again $nrows = scalar @$table; return if ( $nrows < 1 ); # doesn't have poe, or snmp sub-daemon died again $has_poe = 1; $total_avail = 0; $total_consumed = 0; foreach $i ( 1 .. $nrows - 1 ) { next unless ( $table->[ $i ][ 2 ] ); $id = $i - 1; $name = $membernames[ $id ]; $consumed = $table->[ $i ][ 4 ]; if ( $jnxBoxDescr && $version =~ m/^11/ ) { # juniper 11.4 does consumed in milliwatts # juniper 12.3 does consumed in watts # cisco does it in watts $consumed /= 1000; } $avail = $table->[ $i ][ 2 ]; # both do avail in watts $util = $consumed * 100 / $avail; $verbose && printf "%s poe at %0.1f W / %0.0f W = %0.0f %%\n", $name, $consumed, $avail, $util; if ( $util > 80 ) { push @warns, sprintf "%s poe at %0.1f / %0.0f W = %0.0f %%", $name, $consumed, $avail, $util; } $total_avail += $avail; $total_consumed += $consumed; } if ( $total_avail ) { $util = $total_consumed * 100 / $total_avail; } else { $util = -1; } push @oks, sprintf "total poe %0.1f / %0.0f W = %0.1f %%", $total_consumed, $total_avail, $util; push @perfdata, sprintf "poe_used=%0.1f", $total_consumed; # performance data in watts push @perfdata, sprintf "poe_total=%0.0f", $total_avail; # performance data in watts } # check_poe sub check_ports { my( $result, @oids, $oid, $val, $row, $rows, $table, $now, $r, $elapsed, $update_sql, $update_sth, $insert_sql, $insert_sth, $ifIndex, $dot1dBasePort, %ifIndices, $ok, $nrows, @params, $trunk ); $now = time(); # ifTable # 1 ifIndex InterfaceIndex, # 2 ifDescr DisplayString, # 3 ifType IANAifType, # 4 ifMtu Integer32, # 5 ifSpeed Gauge32, # 6 ifPhysAddress PhysAddress, # 7 ifAdminStatus INTEGER, # 1 up, 2 down, 3 testing # 8 ifOperStatus INTEGER, # 9 ifLastChange TimeTicks, # 10 ifInOctets Counter32, # 11 ifInUcastPkts Counter32, # 12 ifInNUcastPkts Counter32, -- deprecated # 13 ifInDiscards Counter32, # 14 ifInErrors Counter32, # 15 ifInUnknownProtos Counter32, # 16 ifOutOctets Counter32, # 17 ifOutUcastPkts Counter32, # 18 ifOutNUcastPkts Counter32, -- deprecated # 19 ifOutDiscards Counter32, # 20 ifOutErrors Counter32, # 21 ifOutQLen Gauge32, -- deprecated # 22 ifSpecific OBJECT IDENTIFIER -- deprecated # # # Cisco 6509s set to get overloaded and truncate the snmp response if we # request too much at once. So do Juniper SRX100s # # PaloAltos can make ifIndecies like 500020263, WTF? # Use snmp_table_multi_index so it's a hash, not an array that eats all of memory. # $table = snmp_table_multi_index( $session, 'ifTable 1/3', "$mib2.2.2", [ 2, 3, 7 ] ); return unless ( $table ); $nrows = scalar keys %$table; $verbose && print "got $nrows rows\n"; foreach $ifIndex ( sort keys %$table ) { $port = clean_port_name( $table->{$ifIndex}[2] ); next if ( ! useful_port( $port ) ); $ifIndex_to_port{ $ifIndex } = $port; $snmp{ $port }->{ 'ifIndex' } = $ifIndex; $snmp{ $port }->{ 'ifType' } = $table->{$ifIndex}[3]; $snmp{ $port }->{ 'ifAdminStatus' } = $table->{$ifIndex}[7]; $snmp{ $port }->{ 'polled' } = $now; } usleep( 500000 ); $table = snmp_table_multi_index( $session, 'ifTable 2/3', "$mib2.2.2", [ 8, 9 ] ); if ( $table ) { foreach $ifIndex ( sort keys %$table ) { $port = $ifIndex_to_port{ $ifIndex }; next if ( ! useful_port( $port ) ); $snmp{ $port }->{ 'ifOperStatus' } = $table->{$ifIndex}[8]; $snmp{ $port }->{ 'ifLastChange' } = $table->{$ifIndex}[9]; if ( $snmp{ $port }->{ 'ifOperStatus' } == 1 ) { $snmp{ $port }->{ 'last_up' } = $now; } else { $snmp{ $port }->{ 'last_up' } = $db{ $port }->{ 'last_up' }; } } } usleep( 500000 ); $table = snmp_table_multi_index( $session, 'ifTable 3/3', "$mib2.2.2", [ 13, 14, 19, 20 ] ); if ( $table ) { foreach $ifIndex ( sort keys %$table ) { $port = $ifIndex_to_port{ $ifIndex }; next if ( ! useful_port( $port ) ); $snmp{ $port }->{ 'ifInDiscards' } = $table->{$ifIndex}[13]; $snmp{ $port }->{ 'ifInErrors' } = $table->{$ifIndex}[14]; $snmp{ $port }->{ 'ifOutDiscards' } = $table->{$ifIndex}[19]; $snmp{ $port }->{ 'ifOutErrors' } = $table->{$ifIndex}[20]; $elapsed = $now - $db{ $port }->{ 'polled' }; if ( $elapsed > 0.0 ) { $snmp{ $port }->{ 'ifInErrors_rate' } = ( $snmp{ $port }->{ 'ifInErrors' } - $db{ $port }->{ 'ifInErrors' } ) / $elapsed / 60.0; $snmp{ $port }->{ 'ifOutErrors_rate' } = ( $snmp{ $port }->{ 'ifOutErrors' } - $db{ $port }->{ 'ifOutErrors' } ) / $elapsed / 60.0; $snmp{ $port }->{ 'ifInDiscards_rate' } = ( $snmp{ $port }->{ 'ifInDiscards' } - $db{ $port }->{ 'ifInDiscards' } ) / $elapsed / 60.0; $snmp{ $port }->{ 'ifOutDiscards_rate' } = ( $snmp{ $port }->{ 'ifOutDiscards' } - $db{ $port }->{ 'ifOutDiscards' } ) / $elapsed / 60.0; } else { $snmp{ $port }->{ 'ifInErrors_rate' } = 0; $snmp{ $port }->{ 'ifOutErrors_rate' } = 0; $snmp{ $port }->{ 'ifInDiscards_rate' } = 0; $snmp{ $port }->{ 'ifOutDiscards_rate' } = 0; } } } # get ifAlias for port descriptions usleep( 500000 ); $result = snmp_walk( $session, 'ifAlias', "$mib2.31.1.1.1.18" ); if ( $result ) { foreach $oid ( keys %$result ) { $val = $result->{ $oid }; $verbose >= 2 && print "$oid = '$val'\n"; next if ( $val eq 'endOfMibView' ); if ( $oid =~ m/.*\.(\d+)$/ ) { $ifIndex = $1; $port = $ifIndex_to_port{ $ifIndex }; $val =~ s/^\s*(.*?)\s*$/$1/; $snmp{ $port }->{ 'ifAlias' } = $val; } } } # get port trunking data usleep( 500000 ); if ( $vendor eq 'Cisco' ) { # enterprises.cisco.ciscoMgmt.ciscoVtpMIB.vtpMIBObjects.vlanTrunkPorts.vlanTrunkPortTable.vlanTrunkPortEntry.vlanTrunkPortDynamicStatus $table = snmp_table( $session, 'vlanTrunkPortTable', "$cisco.9.46.1.6.1", [ 14 ] ); if ( $table ) { #print "vlanTrunkPortTable ", Dumper( $table ); $nrows = scalar( @$table ); foreach $ifIndex ( 1 .. $nrows ) { next if ( ! exists $ifIndex_to_port{ $ifIndex } ); $port = $ifIndex_to_port{ $ifIndex }; next if ( ! $port ); $trunk = ( $table->[$ifIndex][14] == 1 ) ? 1 : 0; $snmp{ $port }->{ 'trunk' } = $trunk; $ntrunks++ if ( $trunk ); } } } # juniper ex or srx elsif ( $vendor eq 'Juniper' && ( $jnxBoxDescr =~ m/Juniper EX|Juniper Virtual Chassis|SRX/i || $sysDescr =~ m/Juniper.*(ex|srx)\d/i ) ) { # juniper EX or virtual chassis switch my( $dot1dBasePort, $dot1dvlan, %ifIndices, %vlanids, %n_tagged_vlans_on_port, %n_untagged_vlans_on_port, $val, $oid, $vlan, $octet, $i, @portlist ); usleep( 500000 ); $result = snmp_walk( $session, 'dot1dBasePortIfIndex', "$mib2.17.1.4.1.2" ); if ( $result ) { foreach $oid ( keys %$result ) { $ifIndex = $result->{ $oid }; $verbose > 1 && print "$oid = $ifIndex\n"; next if ( $ifIndex eq 'endOfMibView' ); if ( $oid =~ m/.*\.(\d+)$/ ) { $dot1dBasePort = $1; $ifIndices{ $dot1dBasePort } = $ifIndex; # ifIndex $port = $ifIndex_to_port{ $ifIndex }; $verbose && print "ifIndex $ifIndex, port $port, dot1dBasePort $dot1dBasePort\n"; $snmp{ $port }->{ 'dot1dBasePort' } = $dot1dBasePort; } } } # need to zero the counts, some logic tests don't work right with undefs foreach $dot1dBasePort ( keys %ifIndices ) { $n_tagged_vlans_on_port{ $dot1dBasePort } = 0; $n_untagged_vlans_on_port{ $dot1dBasePort } = 0; $port = $ifIndex_to_port{ $ifIndex }; if ( $port ) { $snmp{ $port }->{ 'trunk' } = 0; } } # todo: # JUNIPER-IF-MIB::ifJnxInErrors.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxInFrameErrors.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxInQDrops.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxInRunts.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxInGiants.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxInDiscards.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxInHslCrcErrors.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxInHslFifoOverFlows.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxInL3Incompletes.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxInL2ChanErrors.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxInL2MismatchTimeouts.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxInInvalidVCs.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxInFifoErrors.507 = Counter32: 0 # JUNIPER-IF-MIB::ifJnxBucketDrops.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxSramErrors.507 = Counter32: 0 # JUNIPER-IF-MIB::ifJnxOutErrors.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxCollisions.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxCarrierTrans.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxOutQDrops.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxOutAgedErrors.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxOutFifoErrors.507 = Counter32: 0 # JUNIPER-IF-MIB::ifJnxOutHslFifoUnderFlows.507 = Counter64: 0 # JUNIPER-IF-MIB::ifJnxOutHslCrcErrors.507 = Counter32: 0 # Tried to use jnxExVlanPortStatus, or # dot1dPortCapabilities to determine trunks but # they both get confused by voice vlans, and make # everything look like a trunk. # get the list of vlan ids, and the mapping with their internal snmp ids usleep( 100000 ); $result = snmp_walk( $session, 'jnxExVlanTag', "$juniper.3.40.1.5.1.5.1.5" ); if ( $result ) { foreach $oid ( keys %$result ) { next if ( $oid !~ m/\.(\d+)$/ ); $dot1dvlan = $1; $vlan = $result->{ $oid }; $vlanids{ $dot1dvlan } = $vlan; $verbose && print "dot1dvlan $dot1dvlan = vlan tag id $vlan\n"; } # which vlans are allowed out on each port usleep( 100000 ); $result = snmp_walk( $session, "dot1qVlanStaticEgressPorts", "$mib2.17.7.1.4.3.1.2" ); if ( $result ) { foreach $oid ( keys %$result ) { next if ( $oid !~ m/4\.3\.1\.2\.(\d+)$/ ); $dot1dvlan = $1; $verbose && print "$dot1dvlan $result->{ $oid }\n"; @portlist = unpack( "C*", $result->{ $oid } ); $verbose && print "$dot1dvlan @portlist\n"; $dot1dBasePort = 1; foreach $octet ( @portlist ) { #next if ( ! $octet ); foreach $i ( 7, 6, 5, 4, 3, 2, 1, 0 ) { if ( $octet & ( 1 << $i ) ) { $n_tagged_vlans_on_port{ $dot1dBasePort }++; $n_untagged_vlans_on_port{ $dot1dBasePort } = 0; if ( $verbose ) { printf "vlan %d, %d, tagged port %d, %d, %s\n", $dot1dvlan, $vlanids{ $dot1dvlan }, $dot1dBasePort, $ifIndices{ $dot1dBasePort }, $ifIndex_to_port{ $ifIndices{ $dot1dBasePort } }; } } $dot1dBasePort++; } } } # which vlans gets untagged traffic received on each port usleep( 100000 ); $result = snmp_walk( $session, "dot1qVlanStaticUntaggedPorts", "$mib2.17.7.1.4.3.1.4" ); if ( $result ) { foreach $oid ( keys %$result ) { next if ( $oid !~ m/4\.3\.1\.4\.(\d+)$/ ); $dot1dvlan = $1; $verbose > 1 && print "$dot1dvlan $result->{ $oid }\n"; @portlist = unpack( "C*", $result->{ $oid } ); $verbose > 1 && print "$dot1dvlan @portlist\n"; $dot1dBasePort = 1; foreach $octet ( @portlist ) { #next if ( ! $octet ); foreach $i ( 7, 6, 5, 4, 3, 2, 1, 0 ) { if ( $octet & ( 1 << $i ) ) { $n_untagged_vlans_on_port{ $dot1dBasePort }++; if ( $verbose ) { printf "vlan %d, %d, untagged port %d, %d, %s\n", $dot1dvlan, $vlanids{ $dot1dvlan }, $dot1dBasePort, $ifIndices{ $dot1dBasePort }, $ifIndex_to_port{ $ifIndices{ $dot1dBasePort } }; } } $dot1dBasePort++; } } } # now figure out which are "real" trunks foreach $dot1dBasePort ( keys %ifIndices ) { if ( $n_tagged_vlans_on_port{ $dot1dBasePort } > 2 || ( $n_tagged_vlans_on_port{ $dot1dBasePort } >= 1 && $n_untagged_vlans_on_port{ $dot1dBasePort } == 0 ) ) { $ifIndex = $ifIndices{ $dot1dBasePort }; $port = $ifIndex_to_port{ $ifIndex }; if ( $port ) { $verbose && print "$dot1dBasePort $ifIndex $port trunk\n"; $snmp{ $port }->{ 'trunk' } = 1; $ntrunks++; } else { $verbose && print "$dot1dBasePort $ifIndex trunk\n"; } } } } } } } # juniper ex or srx elsif ( $vendor eq 'Juniper' && $jnxBoxDescr =~ m/MX/i ) { # The only way I know to see if it's a trunk is if it's got multiple subinterfaces. # I don't see how to tell if it's vpls. my( $parent, $sub ); $verbose && print "tring to guess Juniper MX trunks\n"; foreach $port ( keys %snmp ) { $verbose && print " port $port\n"; if ( $port =~ m/(.*?)\.(\d+)$/ ) { $parent = $1; $sub = $2; $verbose && print " port $port, parent $parent, sub $sub\n"; if ( exists $snmp{ $parent } ) { $snmp{ $parent }->{ 'subcount' } ++; } } } foreach $port ( keys %snmp ) { $verbose && print " port $port\n"; if ( $snmp{ $port }->{ 'subcount' } > 1 ) { $verbose && print " port $port, subcount $snmp{ $port }->{ subcount }\n"; $snmp{ $port }->{ 'trunk' } = 1; $ntrunks++; } } } elsif ( $vendor eq 'PaloAlto' ) { $verbose && print "PaloAlto, don't know how to get vlan data\n"; } else { $verbose && print "unrecognized device, don't know how to get vlan data\n"; } $verbose && print "ntrunks $ntrunks\n"; # poe status if ( $has_poe ) { my( $groupidx, $portidx, $idx, $nerror, $status, $ndelivering, $nports, $noff, %poeid_to_ifIndex, %poeid_to_port, %groupport_to_port, $nphys ); if ( $vendor eq 'Cisco' ) { # Can get to ifdescr with this sequence: # CISCO-POWER-ETHERNET-EXT-MIB::cpeExtPsePortEntPhyIndex.1.45 = INTEGER: 1051 # ENTITY-MIB::entAliasMappingIdentifier.1051.0 = OID: IF-MIB::ifIndex.10145 # IF-MIB::ifDescr.10145 = STRING: GigabitEthernet0/45 $result = snmp_walk( $session, 'entAliasMappingIdentifier', "$mib2.47.1.3.2.1.2" ); for $oid ( sort keys %$result ) { next unless ( $oid =~ m/2\.(\d+)\.0$/ ); $idx = $1; $val = $result->{ $oid }; next unless ( $val =~ m/2\.2\.1\.1\.(\d+)$/ ); $ifIndex = $1; $poeid_to_ifIndex{ $idx } = $ifIndex; $port = $ifIndex_to_port{ $ifIndex }; $poeid_to_port{ $idx } = $port; } # cpeExtPsePortTable # indexed by groupidx and portidx # 1 cpeExtPsePortEnable INTEGER , # 2 cpeExtPsePortDiscoverMode INTEGER , # 3 cpeExtPsePortDeviceDetected TruthValue, # 4 cpeExtPsePortIeeePd TruthValue, # 5 cpeExtPsePortAdditionalStatus BITS, # 6 cpeExtPsePortPwrMax Unsigned32, # 7 cpeExtPsePortPwrAllocated Unsigned32, # 8 cpeExtPsePortPwrAvailable Unsigned32, # 9 cpeExtPsePortPwrConsumption Unsigned32, # 10 cpeExtPsePortMaxPwrDrawn Unsigned32, # 11 cpeExtPsePortEntPhyIndex EntPhysicalIndexOrZero, # 12 cpeExtPsePortPolicingCapable TruthValue, # 13 cpeExtPsePortPolicingEnable INTEGER # 14 ? INTEGER # 15 ? Gauge32 $table = snmp_table_multi_index( $session, 'cpeExtPsePortTable', "$cisco.9.402.1.2", [ 5, 11 ] ); if ( $table ) { my( $groupportidx, $id, $status ); foreach $groupportidx ( sort keys %$table ) { ( $groupidx, $portidx ) = split( m/\./, $groupportidx ); $status = $table->{ $groupportidx }[ 5 ]; # shows up as a hex string $status = ord( $status ); $idx = $table->{ $groupportidx }[ 11 ]; # shows up as a hex string $port = $poeid_to_port{ $idx }; $groupport_to_port{ $groupportidx } = $port; if ( ! $port ) { $port = ''; } $verbose && print "cpeExtPsePortAdditionalStatus $groupportidx $port $status\n"; if ( ( $status & 0x01 ) != 0 ) { push @warns, "poe port $groupidx.$portidx $port is denied"; } if ( ( $status & 0x02 ) != 0 ) { push @warns, "poe port $groupidx.$portidx $port is overdrawn"; } } } } # mib-2.powerEthernetMIB.pethObjects.pethPsePortTable.pethPsePortEntry.pethPsePortDetectionStatus $nports = 0; $ndelivering = 0; $noff = 0; $nerror = 0; $result = snmp_walk( $session, 'pethPsePortDetectionStatus', "$mib2.105.1.1.1.6" ); if ( $result ) { for $oid ( sort keys %$result ) { $verbose > 1 && print "$oid = $result->{ $oid }\n"; next unless ( $oid =~ m/105\.1\.1\.1\.6\.(\d+)\.(\d+)$/ ); $groupidx = $1; $portidx = $2; $status = $result->{ $oid }; if ( $jnxBoxDescr ) { # I give up, the mapping from poe mib # pethMainPseGroupIndex and pethPsePortIndex to virtual # chassis member and ifIndex, is supposed to come from # pethPdPortTable. But it was never confirmed into the # RFC, so juniper doesn't implement it, nor anything # similar. So I'm going to guess they're 1 less. This # seems to sometimes be wrong on ex3300s. # Juniper indexes from 1 in this mib, even though their port numbers # index from 0. Sigh. $port = sprintf "ge-%d/0/%d", $groupidx - 1, $portidx - 1; # mostly guessing here $snmp{ $port }->{ 'groupidx' } = $groupidx; $snmp{ $port }->{ 'portidx' } = $portidx; } elsif ( exists $groupport_to_port{ "$groupidx,$portidx" } ) { $port = $groupport_to_port{ "$groupidx,$portidx" }; $snmp{ $port }->{ 'groupidx' } = $groupidx; $snmp{ $port }->{ 'portidx' } = $portidx; } else { $port = ''; } $verbose && print "poe port $groupidx.$portidx $port $status ", $pethPsePortDetectionStatus_msgs{ $status }, "\n"; $nports++; if ( $status == 3 ) { # delivering $ndelivering++; } elsif ( $status == 2 ) { # searching, probably nothing plugged in $noff++; } elsif ( $status == 1 ) { # disabled $noff++; } elsif ( $status == 4 || $status == 6 ) { #push @warns, "poe port $groupidx.$portidx $port is $pethPsePortDetectionStatus_msgs{ $status }"; $nerror++; } elsif ( $status == 5 ) { #push @ignores, "poe port $groupidx.$portidx $port is testing"; } if ( $port && exists $snmp{ $port } ) { $snmp{ $port }->{ poeStatus } = $status; } } if ( $nports < 1 ) { push @unknowns, "couldn't get poe port status: " . $session->error; #return; } # number of physical ethernets that might be POE if ( $model =~ m/ex4200/i ) { $nphys = 8; } else { $nphys = scalar( grep m%^(?:ge-\d+/\d+/\d+|fe-\d+/\d+/\d+|g0/\d+|f0/\d+)$%, keys %snmp ); } if ( $verbose ) { print "$nphys physical ports\n"; print "$nports poe ports\n"; print "$ndelivering ports deliveringPower\n"; print "$noff poe off\n"; } if ( $nports < ( $nphys * 0.90 ) ) { push @ignores, "only $nports of $nphys ports have POE status\n"; } push @oks, "$ndelivering ports deliveringPower"; } } # make sure nothing's null foreach $port ( keys %snmp ) { $snmp{ $port }->{ 'ifIndex' } ||= 0; $snmp{ $port }->{ 'ifType' } ||= 0; $snmp{ $port }->{ 'ifAdminStatus' } ||= 0; $snmp{ $port }->{ 'ifOperStatus' } ||= 0; $snmp{ $port }->{ 'ifLastChange' } ||= 0; $snmp{ $port }->{ 'ifInErrors' } ||= 0; $snmp{ $port }->{ 'ifOutErrors' } ||= 0; $snmp{ $port }->{ 'ifInDiscards' } ||= 0; $snmp{ $port }->{ 'ifOutDiscards' } ||= 0; $snmp{ $port }->{ 'ifInErrors_rate' } ||= 0; $snmp{ $port }->{ 'ifOutErrors_rate' } ||= 0; $snmp{ $port }->{ 'ifInDiscards_rate' } ||= 0; $snmp{ $port }->{ 'ifOutDiscards_rate' } ||= 0; $snmp{ $port }->{ 'groupidx' } ||= 0; $snmp{ $port }->{ 'portidx' } ||= 0; # initialize the other stuff to avoid null $snmp{ $port }->{ 'ifAlias' } ||= ''; $snmp{ $port }->{ 'trunk' } ||= 0; $snmp{ $port }->{ 'dot1dBasePort' } ||= 0; $snmp{ $port }->{ 'poeStatus' } ||= 0; $snmp{ $port }->{ 'last_up' } ||= 1; } ######################## # assume certain things are supposed to be up if ( $expect_all ) { foreach $port ( keys %snmp ) { next if ( $port =~ m/\.\d+$/ ); # omit subinterfaces next if ( $ports_to_ignore{ $port } ); if ( $snmp{ $port }->{ 'ifAdminStatus' } == 1 ) { $ports_to_expect_up{ $port } = 1; } } } elsif ( $jnxBoxDescr =~ m/Juniper Virtual Chassis/i ) { # in a virtual chassis, ie stack, make sure all vcp ports are up # anything else is a down stacking connection, or a misconfiguration # # This doesn't work well, because only the master member's vcp ports # show up in the interfaces mib. Sigh. # They do show up in the jnxVirtualChassisPortsTable, # which we check elsewhere. foreach $port ( keys %snmp ) { next if ( $ports_to_ignore{ $port } ); if ( $snmp{ $port }->{ 'ifAdminStatus' } == 1 && $port =~ m/^vcp-/i ) { $ports_to_expect_up{ $port } = 1; } } } # check everything is up that's supposed to be # we want these errors to be emitted first foreach $port ( sort keys %ports_to_expect_up ) { next if ( $ports_to_ignore{ $port } ); check_port( $port, 1 ); } # check each port for other errors foreach $port ( sort keys %snmp ) { next if ( ! $port ); $verbose && printf "port %-15s, expect_up %d, ignore %d, trunk %d, interesting %d\n", $port, $ports_to_expect_up{ $port }, $ports_to_ignore{ $port }, $snmp{ $port }->{ 'trunk' }, interesting_port( $port ); next if ( $ports_to_expect_up{ $port } ); # already did them. next if ( $ports_to_ignore{ $port } ); #next if ( ! interesting_port( $port ) ); check_port( $port, 0 ); } update_db(); } # check_ports sub one_interface { my( $port ) = @_; my( $n, $polled, $now, $elapsed, $ifIndex, $ifEntry, @oids, $result, $ifDescr, $groupidx, $portidx, $pethPsePortDetectionStatus ); # get historical interface data from database $n = load_db_data( $host, $port ); if ( $n == 0 ) { push @unknowns, "no data found for $host $port"; return; } # too old? $polled = $db{ $port }->{ 'polled' }; $now = time(); $elapsed = $now - $db{ $port }->{ 'polled' }; $verbose && print "time elapsed since last poll: $elapsed sec\n"; if ( $elapsed >= $min_poll_time || $force ) { # try to get some fresh data $ifIndex = $db{ $port }->{ 'ifIndex' }; if ( $ifIndex > 0 ) { # ifTable # 1 ifIndex InterfaceIndex, # 2 ifDescr DisplayString, # 3 ifType IANAifType, # 4 ifMtu Integer32, # 5 ifSpeed Gauge32, # 6 ifPhysAddress PhysAddress, # 7 ifAdminStatus INTEGER, # 1 up, 2 down, 3 testing # 8 ifOperStatus INTEGER, # 9 ifLastChange TimeTicks, # 10 ifInOctets Counter32, # 11 ifInUcastPkts Counter32, # 12 ifInNUcastPkts Counter32, -- deprecated # 13 ifInDiscards Counter32, # 14 ifInErrors Counter32, # 15 ifInUnknownProtos Counter32, # 16 ifOutOctets Counter32, # 17 ifOutUcastPkts Counter32, # 18 ifOutNUcastPkts Counter32, -- deprecated # 19 ifOutDiscards Counter32, # 20 ifOutErrors Counter32, # 21 ifOutQLen Gauge32, -- deprecated # 22 ifSpecific OBJECT IDENTIFIER -- deprecated if ( ! $session ) { &start_session(); if ( ! $session ) { return; } } $ifEntry = "$mib2.2.2.1"; @oids = ( "$ifEntry.2.$ifIndex", "$ifEntry.3.$ifIndex", "$ifEntry.7.$ifIndex", "$ifEntry.8.$ifIndex", "$ifEntry.9.$ifIndex", "$ifEntry.13.$ifIndex", "$ifEntry.14.$ifIndex", "$ifEntry.19.$ifIndex", "$ifEntry.20.$ifIndex" ); $result = snmp_get( $session, "ifTable row $ifIndex", \@oids ); $now = time(); if ( ! defined( $result ) ) { $verbose && print "snmp error ", $session->error(), "\n"; push @unknowns, "couldn't get fresh data " . $session->error(); return; } $ifDescr = clean_port_name( $result->{ "$ifEntry.2.$ifIndex" } ); if ( $ifDescr eq $port ) { $snmp{ $port }->{ 'ifIndex' } = $ifIndex; $snmp{ $port }->{ 'ifType' } = $result->{ "$ifEntry.3.$ifIndex" }; $snmp{ $port }->{ 'ifAdminStatus' } = $result->{ "$ifEntry.7.$ifIndex" }; $snmp{ $port }->{ 'ifOperStatus' } = $result->{ "$ifEntry.8.$ifIndex" }; $snmp{ $port }->{ 'ifLastChange' } = $result->{ "$ifEntry.9.$ifIndex" }; $snmp{ $port }->{ 'ifInDiscards' } = $result->{ "$ifEntry.13.$ifIndex" }; $snmp{ $port }->{ 'ifInErrors' } = $result->{ "$ifEntry.14.$ifIndex" }; $snmp{ $port }->{ 'ifOutDiscards' } = $result->{ "$ifEntry.19.$ifIndex" }; $snmp{ $port }->{ 'ifOutErrors' } = $result->{ "$ifEntry.20.$ifIndex" }; $snmp{ $port }->{ 'polled' } = $now; if ( $snmp{ $port }->{ 'ifOperStatus' } == 1 ) { $snmp{ $port }->{ 'last_up' } = $now; } else { $snmp{ $port }->{ 'last_up' } = $db{ $port }->{ 'last_up' }; } $elapsed = $now - $db{ $port }->{ 'polled' }; if ( $elapsed > 0.0 ) { $snmp{ $port }->{ 'ifInErrors_rate' } = ( $snmp{ $port }->{ 'ifInErrors' } - $db{ $port }->{ 'ifInErrors' } ) / $elapsed / 60.0; $snmp{ $port }->{ 'ifOutErrors_rate' } = ( $snmp{ $port }->{ 'ifOutErrors' } - $db{ $port }->{ 'ifOutErrors' } ) / $elapsed / 60.0; $snmp{ $port }->{ 'ifInDiscards_rate' } = ( $snmp{ $port }->{ 'ifInDiscards' } - $db{ $port }->{ 'ifInDiscards' } ) / $elapsed / 60.0; $snmp{ $port }->{ 'ifOutDiscards_rate' } = ( $snmp{ $port }->{ 'ifOutDiscards' } - $db{ $port }->{ 'ifOutDiscards' } ) / $elapsed / 60.0; } else { $snmp{ $port }->{ 'ifInErrors_rate' } = 0; $snmp{ $port }->{ 'ifOutErrors_rate' } = 0; $snmp{ $port }->{ 'ifInDiscards_rate' } = 0; $snmp{ $port }->{ 'ifOutDiscards_rate' } = 0; } # get fresh poe status # default to value from database $snmp{ $port }->{ 'poeStatus' } = $db{ $port }->{ 'poeStatus' }; $pethPsePortDetectionStatus = "$mib2.105.1.1.1.6"; $groupidx = $db{ $port }->{ 'groupidx' }; $portidx = $db{ $port }->{ 'portidx' }; $snmp{ $port }->{ 'groupidx' } = $db{ $port }->{ 'groupidx' }; $snmp{ $port }->{ 'portidx' } = $db{ $port }->{ 'portidx' }; if ( defined $groupidx && defined $portidx ) { @oids = ( "$pethPsePortDetectionStatus.$groupidx.$portidx" ); $result = snmp_get( $session, "pethPsePortDetectionStatus $groupidx $portidx", \@oids ); if ( defined $result && $result->{ "$pethPsePortDetectionStatus.$groupidx.$portidx" } ) { $snmp{ $port }->{ 'poeStatus' } = $result->{ "$pethPsePortDetectionStatus.$groupidx.$portidx" }; $verbose && printf "fresh poeStatus %d %s\n", $snmp{ $port }->{ 'poeStatus' }, $pethPsePortDetectionStatus_msgs{ $snmp{ $port }->{ 'poeStatus' } }; } } $snmp{ $port }->{ 'dot1dBasePort' } = $db{ $port }->{ 'dot1dBasePort' }; $snmp{ $port }->{ 'trunk' } = $db{ $port }->{ 'trunk' }; } else { # oops, ifIndices must have changed # have to go on cached data. push @unknowns, "ifIndex has changed, ifDescr was '$port' now '$ifDescr'"; } } else { # saved ifIndex is invalid # don't trust any other data for this interface push @unknowns, "saved ifIndex is invalid"; return; } } # elapsed < 300 # else use cached data check_port( $port, 1 ); if ( $snmp{ $port }->{ 'ifAdminStatus' } ) { update_db(); } push @oks, $port; if ( $db{ $port }->{ 'ifAlias' } ) { push @oks, $db{ $port }->{ 'ifAlias' }; } } # one_interface sub check_port { my( $port, $expect_up ) = @_; my( $port_data, $a, $o, $rate, $elapsed, $status, $ifAlias ); if ( $verbose ) { print "check_port( $port, $expect_up )\n"; if ( $verbose > 1 ) { printf "-%s %d %d %d %d - %d %d - %d %d\n", $port, $db{ $port }->{ polled }, $db{ $port }->{ ifAdminStatus }, $db{ $port }->{ ifOperStatus }, $db{ $port }->{ ifInErrors }, $db{ $port }->{ ifOutErrors }, $db{ $port }->{ ifInDiscards }, $db{ $port }->{ ifOutDiscards }; if ( $snmp{ $port } ) { printf "+%s %d %d %d %d - %d %d - %d %d\n", $port, $snmp{ $port }->{ ifLastChange }, $snmp{ $port }->{ ifAdminStatus }, $snmp{ $port }->{ ifOperStatus }, $snmp{ $port }->{ ifInErrors }, $snmp{ $port }->{ ifOutErrors }, $snmp{ $port }->{ ifInDiscards }, $snmp{ $port }->{ ifOutDiscards }; } } } if ( $snmp{ $port } ) { # fresh data $port_data = $snmp{ $port }; } else { $port_data = $db{ $port }; } if ( ! exists $port_data->{ 'ifAdminStatus' } ) { push @unknowns, "port $port has no data"; return; } $ifAlias = $port_data->{ 'ifAlias' }; $ifAlias ||= $db{ $port }->{ 'ifAlias' }; # admin and oper state $a = $port_data->{ 'ifAdminStatus' }; $o = $port_data->{ 'ifOperStatus' }; $verbose && print "checking port $port, admin $a, oper $o, ifAlias $ifAlias\n"; # 1 up, 2 down, 3 testing, 4 unknown, 5 dormant, 6 notpresent, 7 lower layer down $verbose && printf " last_up %u %us ago\n", $db{ $port }->{ 'last_up' }, $db{ $port }->{ 'last_up' } - $db{ $port }->{ 'polled' }; if ( $o != 1 && $db{ $port }->{ 'last_up' } < ( $db{ $port }->{ 'polled' } - $oper_down_window ) ) { # it's down, and has been for a long time, # possibly since the switch was installed. # Don't bother alarming about it. $verbose && print "$port long-down since $db{ $port }->{ 'last_up' } $ifAlias\n"; return; } if ( $a == 1 ) { if ( $o == 1 ) { # ok } elsif ( $o == 2 ) { my $interesting = interesting_port( $port ); if ( $expect_up || $interesting ) { push @crits, "$port down $ifAlias"; } if ( $interesting && $snmp{ $port } && $snmp{ $port }->{ ifAdminStatus } == 1 && $snmp{ $port }->{ ifOperStatus } == 2 && $db{ $port }->{ ifOperStatus } == 1 ) { # todo: some sort of history tracking? Look at last N polls? $elapsed = ( $sysUpTime - $snmp{ $port }->{ ifLastChange } ); if ( $elapsed < 360000 ) { push @warns, "$port transitioned down " . ticks_to_str( $elapsed ) . " ago"; } } } elsif ( $o == 3 ) { push @warns, "$port testing $ifAlias"; } elsif ( $o == 5 ) { push @warns, "$port dormant $ifAlias"; } elsif ( $o == 6 ) { push @warns, "$port notpresent $ifAlias"; } elsif ( $o == 7 ) { # ignore this, it's only for subinterfaces #push @warns, "$port lower layer down $ifAlias"; } else { push @unknowns, "$port in unknown state $ifAlias"; } } elsif ( $expect_up ) { if ( $a == 2 ) { push @warns, "$port administratively down $ifAlias"; } elsif ( $a == 3 ) { push @warns, "$port being tested $ifAlias"; } elsif ( $a == 4 ) { push @unknowns, "$port in unknown state $ifAlias"; } elsif ( $a == 5 ) { push @warns, "$port dormant $ifAlias"; } elsif ( $a == 6 ) { push @warns, "$port not present $ifAlias"; } else { push @unknowns, "$port admin status undefined $ifAlias"; } return; } else { return; } if ( exists $snmp{ $port } && $snmp{ $port }->{ 'trunk' } != $db{ $port }->{ 'trunk' } ) { if ( $snmp{ $port }->{ 'trunk' }) { push @ignores, "$port now a trunk $ifAlias"; } else { push @ignores, "$port no longer a trunk $ifAlias"; } } # ifInErrors per minute $rate = $port_data->{ 'ifInErrors_rate' }; $verbose > 1 && print "$port in error rate $rate $ifAlias\n"; if ( $rate >= $warn_rate_errors ) { push @warns, sprintf( "%s in error rate %03.1f/min $ifAlias", $port, $rate ); } # ifOutErrors per minute $rate = $port_data->{ 'ifOutErrors_rate' }; $verbose > 1 && print "$port out error rate $rate $ifAlias\n"; if ( $rate >= $warn_rate_errors ) { push @warns, sprintf( "%s out error rate %03.1f/min $ifAlias", $port, $rate ); } # ifOutDiscards per second $rate = $port_data->{ 'ifInDiscards_rate' }; $verbose > 1 && print "$port in Discard rate $rate $ifAlias\n"; if ( $rate >= $warn_rate_discards ) { push @warns, sprintf( "%s in discard rate %03.1f p/s $ifAlias", $port, $rate ); } # ifOutDiscards per second $rate = $port_data->{ 'ifOutDiscards_rate' }; $verbose > 1 && print "$port out Discard rate $rate $ifAlias\n"; if ( $rate >= $warn_rate_discards ) { push @warns, sprintf( "%s out discard rate %03.1f p/s $ifAlias", $port, $rate ); } # poe $status = $port_data->{ 'poeStatus' }; $verbose > 1 && print "$port poeStatus $status\n"; $verbose && printf "%s poeStatus %d %s\n", $port, $status, $pethPsePortDetectionStatus_msgs{ $status }; if ( $status == 3 ) { # delivering #push @oks, "delivering power $ifAlias"; } elsif ( $status == 2 ) { # searching, probably nothing plugged in } elsif ( $status == 1 ) { # disabled } elsif ( $status == 4 ) { push @warns, "$port poe fault $ifAlias"; } elsif ( $status == 6 ) { #push @warns, "$port poe otherfault $ifAlias"; push @ignores, "$port poe otherfault $ifAlias"; } elsif ( $status == 5 ) { push @ignores, "$port poe testing $ifAlias"; } else { # unknown or undefined in mib, ie xe-0/0/2 won't have one } } # check_port # get historical interface data from database # returns number of rows found # port is optional sub load_db_data { my( $host, $port ) = @_; my( $sql1, @parms1, $sql2, @parms2, $sql, @parms, $rows, $row, $n, $port2 ); $verbose && print "load_db_data( $host, $port )\n"; # load the monitorportopt stuff # If this is a standalone, ie cisco, we have the sw and maybe port. # If this is a stack, we have the name of the stack master, and maybe a port # and this will be in the mangled switch port syntax. # So sw-ne37-211-20 ge-2/0/25 is mangled to sw-ne37-211-22 ge-x/0/25 # But sw-ne37-110-3m f0/23 is in the normal format $sql1 = "select ss2.sw, ss2.idx, m.port, m.opt, m.val from swstk ss join swstk ss2 using( stk ) join monitorportopt m on ss2.sw = m.sw where ss.sw in ( ?"; @parms1 = ( $host ); $sql2 = "select m2.sw, -1, m2.port, m2.opt, m2.val from monitorportopt m2 where m2.sw in ( ?"; @parms2 = ( $host ); $n = $vc_members + 1; # +1 as a precaution if ( $n > $max_membername ) { $n = $max_membername; } foreach $i ( 0 .. $n ) { if ( $membernames[ $i ] && $membernames[ $i ] ne $host ) { $sql1 .= ', ?'; push @parms1, $membernames[ $i ]; $sql2 .= ', ?'; push @parms2, $membernames[ $i ]; } } $sql1 .= " )"; $sql2 .= " )"; if ( $port ) { $port =~ s/^\s*(.*?)\s*$/$1/; $sql1 .= " and m.port = ?"; push @parms1, $port; $sql2 .= " and m2.port = ?"; push @parms2, $port; } $sql = "$sql1\nunion\n$sql2"; @parms = ( @parms1, @parms2 ); $verbose && print "sql $sql, @parms\n"; $rows = $dbh->selectall_arrayref( $sql, undef, @parms ); $n = 0; foreach $row ( @$rows ) { $verbose && print "< ", join( ', ', @$row ), "\n"; my( $sw_mangled, $idx, $port_mangled, $opt, $val ) = @$row; $port_mangled =~ s/^\s*(.*?)\s*$/$1/; if ( $idx >= 0 ) { # we have an idx, so this is a stack and in mangled format ( $port2 = $port_mangled ) =~ s!-x/!-$idx/!; } else { $port2 = $port_mangled; } if ( $opt eq 'monitor' ) { $db{ $port2 }->{ 'monitor' } = $val; if ( $val =~ m/^(?:t|true|y|yes|on|1)\s*$/i ) { $verbose && print "setting \$ports_to_expect_up{ $port2 }\n"; $ports_to_expect_up{ $port2 } = 1; } elsif ( $val =~ m/^(?:f|false|n|no|off|0)\s*$/i ) { $verbose && print "setting \$ports_to_ignore{ $port2 }\n"; $ports_to_ignore{ $port2 } = 1; } } } $verbose && print "loaded $n rows from monitorportopt database\n"; # get historical interface data from database $sql = "select port, ifAlias, ifIndex, dot1dBasePort, trunk, ifAdminStatus, ifOperStatus, ifInErrors, ifOutErrors, ifInDiscards, ifOutDiscards, ifInErrors_rate, ifOutErrors_rate, ifInDiscards_rate, ifOutDiscards_rate, groupidx, portidx, poeStatus, polled, last_up from swports where sw = ?"; @parms = ( $host ); if ( $port ) { $sql .= " and port = ?"; push @parms, $port; } $rows = $dbh->selectall_arrayref( $sql, undef, @parms ); $n = 0; foreach $row ( @$rows ) { $verbose && print "< ", join( ', ', @$row ), "\n"; $port = $row->[ 0 ]; $port =~ s/^\s*(.*?)\s*$/$1/; $db{ $port }->{ 'ifAlias' } = $row->[ 1 ]; $db{ $port }->{ 'ifIndex' } = $row->[ 2 ]; $db{ $port }->{ 'dot1dBasePort' } = $row->[ 3 ]; $db{ $port }->{ 'trunk' } = $row->[ 4 ]; $db{ $port }->{ 'ifAdminStatus' } = $row->[ 5 ]; $db{ $port }->{ 'ifOperStatus' } = $row->[ 6 ]; $db{ $port }->{ 'ifInErrors' } = $row->[ 7 ]; $db{ $port }->{ 'ifOutErrors' } = $row->[ 8 ]; $db{ $port }->{ 'ifInDiscards' } = $row->[ 9 ]; $db{ $port }->{ 'ifOutDiscards' } = $row->[ 10 ]; $db{ $port }->{ 'ifInErrors_rate' } = $row->[ 11 ]; $db{ $port }->{ 'ifOutErrors_rate' } = $row->[ 12 ]; $db{ $port }->{ 'ifInDiscards_rate' } = $row->[ 13 ]; $db{ $port }->{ 'ifOutDiscards_rate' } = $row->[ 14 ]; $db{ $port }->{ 'groupidx' } = $row->[ 15 ]; $db{ $port }->{ 'portidx' } = $row->[ 16 ]; $db{ $port }->{ 'poeStatus' } = $row->[ 17 ]; $db{ $port }->{ 'polled' } = $row->[ 18 ]; $db{ $port }->{ 'last_up' } = $row->[ 19 ]; $n++; } $verbose && print "loaded $n rows from swports database\n"; return $n; } sub update_db { my( $update_sql, $update_sth, $port, @params, $r, $insert_sql, $insert_sth ); $verbose && print "update_db()\n"; ######################## # update the database # prepare the most common sql update statement $update_sql = "update swports set ifAlias = ?, ifIndex = ?, dot1dBasePort = ?, trunk = ?, ifAdminStatus = ?, ifOperStatus = ?, ifInErrors = ?, ifOutErrors = ?, ifInDiscards = ?, ifOutDiscards = ?, ifInErrors_rate = ?, ifOutErrors_rate = ?, ifInDiscards_rate = ?, ifOutDiscards_rate = ?, groupidx = ?, portidx = ?, poeStatus = ?, polled = ?, last_up = ? where sw = ? and port = ?"; $verbose && print "update sql $update_sql\n"; $update_sth = $dbh->prepare( $update_sql ); foreach $port ( sort keys %snmp ) { next if ( ! useful_port( $port ) ); next unless ( $snmp{ $port }->{ 'ifAdminStatus' } ); if ( exists $db{ $port } && $db{ $port }->{ 'polled' } ) { @params = ( $snmp{ $port }->{ 'ifAlias' }, $snmp{ $port }->{ 'ifIndex' }, $snmp{ $port }->{ 'dot1dBasePort' }, $snmp{ $port }->{ 'trunk' }, $snmp{ $port }->{ 'ifAdminStatus' }, $snmp{ $port }->{ 'ifOperStatus' }, $snmp{ $port }->{ 'ifInErrors' }, $snmp{ $port }->{ 'ifOutErrors' }, $snmp{ $port }->{ 'ifInDiscards' }, $snmp{ $port }->{ 'ifOutDiscards' }, $snmp{ $port }->{ 'ifInErrors_rate' }, $snmp{ $port }->{ 'ifOutErrors_rate' }, $snmp{ $port }->{ 'ifInDiscards_rate' }, $snmp{ $port }->{ 'ifOutDiscards_rate' }, $snmp{ $port }->{ 'groupidx' }, $snmp{ $port }->{ 'portidx' }, $snmp{ $port }->{ 'poeStatus' }, $snmp{ $port }->{ 'polled' }, $snmp{ $port }->{ 'last_up' }, $host, $port, ); $verbose && print "sql $update_sql, ", join( ", ", @params ), "\n"; if ( ! $noop ) { $r = $update_sth->execute( @params ); $verbose && print "updated returned $r\n"; if ( $r != 1 ) { push @unknowns, "can't update database for $host $port: " . $DBI::errstr; } } } else { if ( ! $insert_sth ) { $insert_sql = "insert into swports ( sw, port, ifAlias, ifIndex, dot1dBasePort, trunk, ifAdminStatus, ifOperStatus, ifInErrors, ifOutErrors, ifInDiscards, ifOutDiscards, ifInErrors_rate, ifOutErrors_rate, ifInDiscards_rate, ifOutDiscards_rate, groupidx, portidx, poeStatus, polled, last_up ) values ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )"; $verbose && print "insert sql $insert_sql\n"; $insert_sth = $dbh->prepare( $insert_sql ); } @params = ( $host, $port, $snmp{ $port }->{ 'ifAlias' }, $snmp{ $port }->{ 'ifIndex' }, $snmp{ $port }->{ 'dot1dBasePort' }, $snmp{ $port }->{ 'trunk' }, $snmp{ $port }->{ 'ifAdminStatus' }, $snmp{ $port }->{ 'ifOperStatus' }, $snmp{ $port }->{ 'ifInErrors' }, $snmp{ $port }->{ 'ifOutErrors' }, $snmp{ $port }->{ 'ifInDiscards' }, $snmp{ $port }->{ 'ifOutDiscards' }, $snmp{ $port }->{ 'ifInErrors_rate' }, $snmp{ $port }->{ 'ifOutErrors_rate' }, $snmp{ $port }->{ 'ifInDiscards_rate' }, $snmp{ $port }->{ 'ifOutDiscards_rate' }, $snmp{ $port }->{ 'groupidx' }, $snmp{ $port }->{ 'portidx' }, $snmp{ $port }->{ 'poeStatus' }, $snmp{ $port }->{ 'polled' }, $snmp{ $port }->{ 'last_up' } ); $verbose && print "sql $insert_sql, ", join( ", ", @params ), "\n"; if ( ! $noop ) { $r = $insert_sth->execute( @params ); $verbose && print "insert returned $r\n"; if ( $r != 1 ) { push @unknowns, "can't insert into database for $host $port: " . $DBI::errstr; } } } } } sub useful_port { my( $port ) = @_; if ( ! $port ) { return 0; } $port =~ s/^\s*(.*?)\s*$/$1/; if ( $ports_to_ignore{ $port } ) { return 0; } elsif ( $port =~ m/^(?:bme\d|cbp\d|demux\d|dsc|em\d|fxp\d|gre|gr-|ipip|ip-|irb|lc-|loopback|lo\d|lsi|lt-|me\d|mtun|mt-|netflow|null|pc-|pd-|pe-|pfe-|pfh-|pimd|pime|pip|pp\d|ppd\d|ppe\d|Serial|sp-|st\d|tap|unrouted|ut-|vlan|Voice|vt-)/i ) { return 0; } return 1; } # is this an interesting port, ie not a loopback or other wierd # thing we don't care about # return 1 for interesting, 0 for boring sub interesting_port { my( $port ) = @_; return 0 if ( ! $port ); $port =~ s/^\s*(.*?)\s*$/$1/; return 0 if ( ! useful_port( $port ) ); return 0 if ( $ports_to_ignore{ $port } ); if ( $check_trunks && exists $snmp{ $port } && $snmp{ $port }->{ 'trunk' } == 1 ) { return 1; } elsif ( $check_trunks && exists $db{ $port } && $db{ $port }->{ 'trunk' } == 1 ) { return 1; } elsif ( exists $ports_to_expect_up{ $port } ) { return 1; } return 0; } sub clean_port_name { my( $port ) = @_; if ( $port ) { $port =~ s/^\s*(.*?)\s*$/$1/; $port =~ s/TenGigabitEthernet/t/; $port =~ s/GigabitEthernet/g/; $port =~ s/FastEthernet/f/; $port =~ s/Ethernet/e/; } return $port; } ########################################### sub snmp_get { my( $session, $name, $oids ) = @_; my( $result, $oid, $val ); $verbose && print "snmp_get $name\n"; $result = $session->get_request( -varbindlist => $oids ); if ( ! defined( $result ) ) { #warn "snmpget error: ", $session->error(), "\n"; $verbose && print "snmpget error: ", $session->error(), "\n"; return undef; } foreach $oid ( @$oids ) { if ( ! exists $result->{ $oid } ) { #warn "snmpget error: requested oid not in response\n"; $verbose && print " $oid: no response\n"; $result->{ $oid } = undef; } elsif ( $result->{ $oid } eq 'noSuchInstance' ) { $verbose && print " $oid: noSuchInstance\n"; $result->{ $oid } = undef; } elsif ( $result->{ $oid } eq 'noSuchObject' ) { $verbose && print " $oid: noSuchObject\n"; $result->{ $oid } = undef; } elsif ( $verbose ) { $val = $result->{ $oid }; $val =~ tr/\040-\176//cd; print " $oid: $val\n"; } } return $result; } sub snmp_get_one { my( $session, $name, $oid ) = @_; my( @oids, $result, $oid2, $val ); $verbose && print "snmp_get $name\n"; @oids = ( $oid ); $result = $session->get_request( -varbindlist => \@oids ); if ( ! defined( $result ) ) { #warn "snmpget error: ", $session->error(), "\n"; return undef; } if ( ! exists $result->{ $oid } ) { #warn "snmpget error: requested oid not in response\n"; $verbose && print " $oid: no response\n"; return undef; } elsif ( $result->{ $oid } eq 'noSuchInstance' ) { $verbose && print " $oid: noSuchInstance\n"; return undef; } elsif ( $result->{ $oid } eq 'noSuchObject' ) { $verbose && print " $oid: noSuchObject\n"; return undef; } if ( $verbose ) { $val = $result->{ $oid }; $val =~ tr/\040-\176//cd; print " $oid: $val\n"; } return $result->{ $oid }; } sub snmp_walk { my( $session, $name, $baseoid ) = @_; my( $result ); $verbose && print "walking $name\n"; $result = $session->get_table( -baseoid => $baseoid, -maxrepetitions => $maxrepetitions ); #print "session error ", $session->error(), "\n"; if ( ! defined( $result ) && $session->error() !~ m/Requested table is empty|Requested entries are empty or do not exist/ ) { $verbose && printf( "error walking $name table on %s: %s\n", $session->hostname, $session->error() ); #push @unknowns, sprintf( "error walking $name table on %s: %s", # $session->hostname, $session->error() ); return undef; } if ( $verbose > 1 ) { my( $oid, $val ); foreach $oid ( sort keys %$result ) { $val = $result->{ $oid }; print "$oid = $val\n"; } } return $result; } # walk a table # With the optional columns parameter, just walk those selected columns out of it. # Return a pointer to a 2d array. sub snmp_table { my( $session, $name, $baseoid, $columns ) = @_; my( $result, $rows, $oid, $val, $col, $row, $data, @oids, $maxmsgsize ); $verbose && print "walking $name table $baseoid\n"; if ( defined $columns ) { foreach $col ( @$columns ) { push @oids, "$baseoid.1.$col"; } } else { push @oids, $baseoid; } $data = []; foreach $maxmsgsize ( $prefered_maxmsgsize, 1472, 1600, 1800, 2048, 4096, 9000 ) { $session->max_msg_size( $maxmsgsize ); $verbose > 1 && print "snmp get_entries tring maxmsgsize $maxmsgsize\n"; $result = $session->get_entries( -columns => \@oids, -maxrepetitions => $maxrepetitions ); #$verbose > 1 && print "get_entries returned $result\n"; if ( defined( $result ) ) { last; } elsif ( $session->error() =~ m/Requested table is empty|Requested entries are empty or do not exist/i ) { # ok return $data; } elsif ( $session->error() =~ m/Message size exceeded buffer maxMsgSize/ ) { # retry with different size $verbose && print "snmp get_entries failed with maxmsgsize $maxmsgsize\n"; } else { $verbose && print "snmp get_entries failed with error ", $session->error(), "\n"; last; } # else try a bigger maxmessage size } if ( ! defined( $result ) ) { $verbose && printf "error walking $name table on %s: %s\n", $session->hostname, $session->error(); #push @unknowns, sprintf "error walking $name table on %s: %s", # $session->hostname, $session->error(); return undef; } foreach $oid ( sort keys %$result ) { $val = $result->{ $oid }; $verbose > 1 && print "$oid = $val\n"; next if ( $val eq 'endOfMibView' ); if ( $oid =~ m/.*\.(\d+)\.(\d+)$/ ) { $col = $1; $row = $2; $data->[$row][$col] = $val; } } if ( scalar( @$data ) < 1 ) { $verbose && printf "no rows in $name table on %s: %s", $session->hostname, $session->error(); #push @unknowns, sprintf "no rows in $name table on %s: %s", # $session->hostname, $session->error(); return undef; } #$verbose && print "returning data from snmp_table()\n"; return $data; } # walk a table where the "row" index is actually multiple levels in the oid # ie a 3 part index sub snmp_table_multi_index { my( $session, $name, $baseoid, $columns ) = @_; my( $result, $rows, $oid, $val, $col, $row, $data, @oids, $maxmsgsize ); $verbose && print "walking $name table\n"; if ( defined $columns ) { foreach $col ( @$columns ) { push @oids, "$baseoid.1.$col"; } } else { push @oids, $baseoid; } $data = {}; foreach $maxmsgsize ( $prefered_maxmsgsize, 1472, 1600, 1800, 2048, 4096, 9000 ) { $session->max_msg_size( $maxmsgsize ); $verbose > 1 && print "snmp get_entries tring maxmsgsize $maxmsgsize\n"; $result = $session->get_entries( -columns => \@oids, -maxrepetitions => $maxrepetitions ); if ( defined( $result ) ) { last; } elsif ( $session->error() =~ m/Requested table is empty|Requested entries are empty or do not exist/ ) { # ok return $data; } elsif ( $session->error() !~ m/Message size exceeded buffer maxMsgSize/ ) { last; } } if ( ! defined( $result ) ) { $verbose && printf "error walking $name table on %s: %s\n", $session->hostname, $session->error(); #push @unknowns, sprintf "error walking $name table on %s: %s", # $session->hostname, $session->error(); return undef; } foreach $oid ( sort keys %$result ) { $val = $result->{ $oid }; $verbose > 1 && print "$oid = $val\n"; next if ( $val eq 'endOfMibView' ); if ( $oid =~ m/^$baseoid\.1\.(\d+)\.([\d\.]+)$/ ) { $col = $1; $row = $2; $data->{$row}[$col] = $val; } } if ( scalar( keys %$data ) < 1 ) { $verbose && printf "no rows in $name table on %s: %s", $session->hostname, $session->error(); #push @unknowns, sprintf "no rows in $name table on %s: %s", # $session->hostname, $session->error(); return undef; } return $data; } sub ticks_to_str { my( $ticks ) = @_; my( @intervals, @letters, $interval, $str, $i, $n, $started ); @intervals = ( 60480000, 8640000, 360000, 6000, 100, ); @letters = ( 'w ', 'd ', 'h ', 'm ', 's', ); $str = ''; for ( $i = 0; $i < 5; $i++ ) { $interval = $intervals[ $i ]; if ( $ticks >= $interval || $started ) { $n = int( $ticks / $interval ); $ticks -= $n * $interval; $str .= sprintf( "%u%s", $n, $letters[ $i ] ); $started = 1; # show days in 3weeks 0days 3hours } } return $str; } # ssh to the switch, run a command and grab the result. # Can't do this entirely in here, because we need to use Expect to give the # password to ssh, and the embedded Perl environment hates the Expect # module. So we use an external helper script. # sub ssh_cmd { my( $cmd ) = @_; my( $exp, $output ); $cmd = "/usr/local/nagios/libexec/ssh_cmd -H $host -U nagios -P $swpasswd_file '$cmd' 2>&1 |"; $verbose && print "ssh cmd: $cmd\n"; if ( ! open( pH, $cmd ) ) { push @unknowns, "can't run '$cmd': $!"; return ''; } $output = ''; while ( ) { $output .= $_; } close pH; $verbose && print "ssh output: $output\n"; return $output; } sub db_do { my( $sql, @params ) = @_; my $r = 1; $verbose && print "sql $sql, ", join( ', ', @params ), "\n"; if ( ! $noop ) { $r = $dbh->do( $sql, undef, @params ); if ( $r < 1 ) { push @unknowns, "couldn't do $sql '$sql', " . join( ', ', @params, ) . ": " . $dbh->errstr . "\n"; } } return $r; } sub get_passwd{ my( $file ) = @_; my( $password ); if ( ! open( pwfH, $file ) ) { push @unknowns, "can't open $file: $!\n"; return undef; } $password = ; chomp( $password ); close pwfH; return $password }