find instances with slaves stuck in WFConnection

[gnt-info] / gnt-monitor

diff --git a/gnt-monitor b/gnt-monitor
index 36a9751..3f1dacc 100755 (executable)
--- a/gnt-monitor
+++ b/gnt-monitor
@@ -3,12 +3,22 @@ use warnings;
 use strict;
 use autodie;
 use POSIX;
+use Time::HiRes qw(time sleep);
 
 my $DEBUG = $ENV{DEBUG} || 0;
-my $INFLUX = 'http://10.60.0.89:8186/write';
+my $INFLUX = $ENV{INFLUX} || 'http://10.80.3.89:8086/write?db=gnt';
+my $INTERVAL = $ENV{INTERVAL} || 1;
+my $DC = $ENV{DC};
+my $RACK = $ENV{RACK};
 
-my $hostname = `hostname -s`;
-chomp $hostname;
+my $node = `hostname -s`;
+chomp $node;
+
+my $location = '';
+$location .= qq{,dc=$DC} if $DC;
+$location .= qq{,rack=$RACK} if $RACK;
+
+warn $0 = "gnt-monitor\@$node$location $INFLUX $INTERVAL $DEBUG";
 
 use Data::Dump;
 sub XXX { $DEBUG ? warn "XXX ",Data::Dump::dump( @_ ) : {} };
@@ -18,6 +28,11 @@ my $last;
 
 while(1) {
 
+my $skip;
+
+my $begin_t = time();
+my $t = int( time() * 1000_000_000 );
+
 foreach my $instance ( glob '/var/run/ganeti/kvm-hypervisor/pid/*' ) {
 
        open(my $fh, '<', $instance);
@@ -25,6 +40,12 @@ foreach my $instance ( glob '/var/run/ganeti/kvm-hypervisor/pid/*' ) {
 
        $instance =~ s{^.*/}{};
 
+       if ( ! -d "/proc/$pid" ) {
+               $skip->{$instance}++;
+               XXX $skip;
+               next;
+       }
+
        my $vcpu = $last->{$instance}->{vcpu};
 
        if ( ! $vcpu ) {
@@ -43,8 +64,9 @@ foreach my $instance ( glob '/var/run/ganeti/kvm-hypervisor/pid/*' ) {
        if ( my $last_gtime = $last->{$instance}->{gtime} ) {
                my $clock_ticks = POSIX::sysconf( &POSIX::_SC_CLK_TCK ); # clock ticks per second
 
-               my $cpu = ( ( $gtime - $last_gtime ) * 100 ) / ( $clock_ticks * $vcpu );
+               my $cpu = ( ( $gtime - $last_gtime ) * 100 ) / ( $clock_ticks * $vcpu ) / $INTERVAL;
                $stat->{$instance}->{cpu} = $cpu;
+               $stat->{$instance}->{ticks} = ($gtime - $last_gtime) / $INTERVAL;
        }
 
        $last->{$instance}->{gtime} = $gtime;
@@ -53,60 +75,122 @@ foreach my $instance ( glob '/var/run/ganeti/kvm-hypervisor/pid/*' ) {
 
 foreach my $glob ( glob '/var/run/ganeti/instance-disks/*' ) {
        my ( $instance, $disk ) = split(/:/,$glob,2);
-       my $dev = readlink $glob;
        $instance =~ s{^.*/}{};
+
+       next unless exists $stat->{$instance};
+
+       my $dev = readlink $glob;
        $dev =~ s{^.*dev/}{};
+
+       if ( ! -e "/sys/class/block/$dev" ) {
+               $skip->{$instance}++;
+               XXX $skip;
+               next;
+       }
+
        open( my $fh, '<', "/sys/class/block/$dev/stat" );
        my $v = <$fh>; chomp $v; $v =~ s/^\s+//;
        my @s = split(/\s+/, $v );
        # https://www.kernel.org/doc/Documentation/block/stat.txt
        my $d = {
-               read_io => $s[0],
-               read_bytes => $s[2] * 512,
-               read_wait => $s[3],
-               write_io => $s[4],
-               write_bytes => $s[6] * 512,
-               write_wait => $s[7],
+               read_io => $s[0] / $INTERVAL,
+               read_bytes => $s[2] * 512 / $INTERVAL,
+               read_wait => $s[3] / $INTERVAL,
+               write_io => $s[4] / $INTERVAL,
+               write_bytes => $s[6] * 512 / $INTERVAL,
+               write_wait => $s[7] / $INTERVAL,
        };
        if ( my $l = $last->{$instance}->{disk}->[$disk] ) {
                my $delta;
                $delta->{$_} = $d->{$_} - $l->{$_} foreach keys %$d;
                $stat->{$instance}->{disk}->[$disk] = $delta;
+               $stat->{$instance}->{disk}->[$disk]->{dev} = $dev;
+               $stat->{$instance}->{disk}->[$disk]->{disk} = $disk;
        }
        $last->{$instance}->{disk}->[$disk] = $d;
-       $stat->{$instance}->{disk}->[$disk]->{dev} = $dev;
 
 }
 
-foreach my $instance ( glob '/var/run/ganeti/kvm-hypervisor/nic/*' ) {
-       foreach my $nic ( glob "$instance/*" ) {
+foreach my $full_instance ( glob '/var/run/ganeti/kvm-hypervisor/nic/*' ) {
+       my $instance = $full_instance;
+       $instance =~ s{^.*/}{};
+       next unless exists $stat->{$instance};
+
+       foreach my $nic ( glob "$full_instance/*" ) {
                open(my $fh, '<', $nic);
                my $dev = <$fh>;
+
+               next unless -e "/sys/class/net/$dev";
+
                $nic =~ s{^.*/}{};
-               $instance =~ s{^.*/}{};
-               $stat->{$instance}->{nic}->[$nic]->{dev} = $dev;
-               my $vlan = readlink "/sys/class/net/$dev/master";
-               $vlan =~ s/^.*br//;
-               $stat->{$instance}->{nic}->[$nic]->{vlan} = $vlan;
 
                my $d;
 
                foreach my $f (qw( rx_bytes tx_bytes rx_packets tx_packets )) {
                        open( my $fh, '<', "/sys/class/net/$dev/statistics/$f" );
                        my $v = <$fh>; chomp $v;
-                       $d->{$f} = $v;
+                       $d->{$f} = $v / $INTERVAL;
                }
                if ( my $l = $last->{$instance}->{nic}->[$nic] ) {
                        $stat->{$instance}->{nic}->[$nic]->{$_} = $d->{$_} - $l->{$_} foreach keys %$d;
+                       $stat->{$instance}->{nic}->[$nic]->{dev} = $dev;
+                       if ( -e "/sys/class/net/$dev/master" ) {
+                               my $vlan = readlink "/sys/class/net/$dev/master";
+                               $vlan =~ s/^.*br//;
+                               $stat->{$instance}->{nic}->[$nic]->{vlan} = $vlan;
+                       }
                }
                $last->{$instance}->{nic}->[$nic] = $d;
 
        }
 }
 
-XXX $stat;
+#XXX $stat;
 #XXX $last;
 
-sleep 1;
+sub dump4influx {
+       my $hash = shift;
+       my @v;
+       foreach my $k ( keys %$hash ) {
+               my $v = $hash->{$k};
+               my ( $d, $s ) = $v =~ m/^\d+(\.\d+)?$/i ? ( '',  ''  ) : # float
+                                                     ( '"' ,''  ) ; # string
+
+               push @v, "$k=$d$v$d$s";
+       }
+       my $i = join(',', @v);
+       return $i;
+}
+
+open(my $fh, '>', '/dev/shm/gnt-monitor.influx');
+
+foreach my $instance ( keys %$stat ) {
+
+       next if $skip->{$instance};
+
+       print $fh qq{ganeti_cpu,node=$node,instance=$instance$location cpu=$stat->{$instance}->{cpu},ticks=$stat->{$instance}->{ticks} $t\n};
+
+       foreach my $disk ( @{ $stat->{$instance}->{disk} } ) {
+               print $fh qq{ganeti_disk,node=$node,instance=$instance$location },dump4influx( $disk ), " $t\n";
+       }
+
+       foreach my $nic ( @{ $stat->{$instance}->{nic} } ) {
+               my $vlan = delete $nic->{vlan};
+               $vlan = 0 unless defined $vlan;
+               print $fh qq{ganeti_nic,node=$node,instance=$instance,vlan=${vlan}$location },dump4influx( $nic ), " $t\n";
+       }
+
+}
+
+close($fh);
+
+if ( system( 'curl', '-s', '-XPOST', $INFLUX, '--data-binary', '@/dev/shm/gnt-monitor.influx' ) == 0 ) {
+} else {
+       rename '/dev/shm/gnt-monitor.influx', '/dev/shm/gnt-monitor.influx.bug.' . scalar glob '/dev/shm/gnt-monitor.influx.bug.*';
+       warn "curl failed: $? $!";
+}
+
+my $dt = time() - $begin_t;
+sleep $INTERVAL - $dt if $dt > 0 && $dt < $INTERVAL;
 } #/while