X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;f=gnt-monitor;h=3f1dacc5288136f3a8d9cb0fa295b9816efd9d6d;hb=04a9a9bf77f94913d7956a52f1c9c0a293f66776;hp=c8df02c34319a5ebcc2978eafb738c21b315f88c;hpb=8b413ad2a3070aec2fadc4d54609d35334fab98e;p=gnt-info diff --git a/gnt-monitor b/gnt-monitor index c8df02c..3f1dacc 100755 --- a/gnt-monitor +++ b/gnt-monitor @@ -2,54 +2,195 @@ use warnings; use strict; use autodie; +use POSIX; +use Time::HiRes qw(time sleep); my $DEBUG = $ENV{DEBUG} || 0; -my $INFLUX = 'http://10.60.0.89:8186/write'; +my $INFLUX = $ENV{INFLUX} || 'http://10.80.3.89:8086/write?db=gnt'; +my $INTERVAL = $ENV{INTERVAL} || 1; +my $DC = $ENV{DC}; +my $RACK = $ENV{RACK}; -my $hostname = `hostname -s`; -chomp $hostname; +my $node = `hostname -s`; +chomp $node; + +my $location = ''; +$location .= qq{,dc=$DC} if $DC; +$location .= qq{,rack=$RACK} if $RACK; + +warn $0 = "gnt-monitor\@$node$location $INFLUX $INTERVAL $DEBUG"; use Data::Dump; sub XXX { $DEBUG ? warn "XXX ",Data::Dump::dump( @_ ) : {} }; my $stat; +my $last; + +while(1) { + +my $skip; + +my $begin_t = time(); +my $t = int( time() * 1000_000_000 ); + +foreach my $instance ( glob '/var/run/ganeti/kvm-hypervisor/pid/*' ) { + + open(my $fh, '<', $instance); + my $pid = <$fh>; chomp $pid; + + $instance =~ s{^.*/}{}; + + if ( ! -d "/proc/$pid" ) { + $skip->{$instance}++; + XXX $skip; + next; + } + + my $vcpu = $last->{$instance}->{vcpu}; + + if ( ! $vcpu ) { + foreach my $fd ( glob "/proc/$pid/fd/*" ) { + $vcpu++ if -l $fd && readlink($fd) =~ m/kvm-vcpu/; + } + + $last->{$instance}->{vcpu} = $vcpu; + } + + # https://www.kernel.org/doc/Documentation/filesystems/proc.txt + open($fh, '<', "/proc/$pid/stat"); + my $line = <$fh>; chomp $line; + my $gtime = (split(/\s+/,$line))[42]; # guest time of the task in jiffies + + if ( my $last_gtime = $last->{$instance}->{gtime} ) { + my $clock_ticks = POSIX::sysconf( &POSIX::_SC_CLK_TCK ); # clock ticks per second + + my $cpu = ( ( $gtime - $last_gtime ) * 100 ) / ( $clock_ticks * $vcpu ) / $INTERVAL; + $stat->{$instance}->{cpu} = $cpu; + $stat->{$instance}->{ticks} = ($gtime - $last_gtime) / $INTERVAL; + } + + $last->{$instance}->{gtime} = $gtime; +} + foreach my $glob ( glob '/var/run/ganeti/instance-disks/*' ) { my ( $instance, $disk ) = split(/:/,$glob,2); - my $dev = readlink $glob; $instance =~ s{^.*/}{}; + + next unless exists $stat->{$instance}; + + my $dev = readlink $glob; $dev =~ s{^.*dev/}{}; - $stat->{$instance}->{disk}->[$disk]->{dev} = $dev; + + if ( ! -e "/sys/class/block/$dev" ) { + $skip->{$instance}++; + XXX $skip; + next; + } + open( my $fh, '<', "/sys/class/block/$dev/stat" ); my $v = <$fh>; chomp $v; $v =~ s/^\s+//; my @s = split(/\s+/, $v ); # https://www.kernel.org/doc/Documentation/block/stat.txt - $stat->{$instance}->{disk}->[$disk]->{read_io} = $s[0]; - $stat->{$instance}->{disk}->[$disk]->{read_bytes} = $s[2] * 512; - $stat->{$instance}->{disk}->[$disk]->{read_wait} = $s[3]; - $stat->{$instance}->{disk}->[$disk]->{write_io} = $s[4]; - $stat->{$instance}->{disk}->[$disk]->{write_bytes} = $s[6] * 512; - $stat->{$instance}->{disk}->[$disk]->{write_wait} = $s[7]; + my $d = { + read_io => $s[0] / $INTERVAL, + read_bytes => $s[2] * 512 / $INTERVAL, + read_wait => $s[3] / $INTERVAL, + write_io => $s[4] / $INTERVAL, + write_bytes => $s[6] * 512 / $INTERVAL, + write_wait => $s[7] / $INTERVAL, + }; + if ( my $l = $last->{$instance}->{disk}->[$disk] ) { + my $delta; + $delta->{$_} = $d->{$_} - $l->{$_} foreach keys %$d; + $stat->{$instance}->{disk}->[$disk] = $delta; + $stat->{$instance}->{disk}->[$disk]->{dev} = $dev; + $stat->{$instance}->{disk}->[$disk]->{disk} = $disk; + } + $last->{$instance}->{disk}->[$disk] = $d; + } -foreach my $instance ( glob '/var/run/ganeti/kvm-hypervisor/nic/*' ) { - foreach my $nic ( glob "$instance/*" ) { - open(my $fh, '<', "$nic"); +foreach my $full_instance ( glob '/var/run/ganeti/kvm-hypervisor/nic/*' ) { + my $instance = $full_instance; + $instance =~ s{^.*/}{}; + next unless exists $stat->{$instance}; + + foreach my $nic ( glob "$full_instance/*" ) { + open(my $fh, '<', $nic); my $dev = <$fh>; + + next unless -e "/sys/class/net/$dev"; + $nic =~ s{^.*/}{}; -XXX $nic; - $instance =~ s{^.*/}{}; - $stat->{$instance}->{nic}->[$nic]->{dev} = $dev; - my $vlan = readlink "/sys/class/net/$dev/master"; - $vlan =~ s/^.*br//; - $stat->{$instance}->{nic}->[$nic]->{vlan} = $vlan; + + my $d; + foreach my $f (qw( rx_bytes tx_bytes rx_packets tx_packets )) { open( my $fh, '<', "/sys/class/net/$dev/statistics/$f" ); my $v = <$fh>; chomp $v; - $stat->{$instance}->{nic}->[$nic]->{$f} = $v; + $d->{$f} = $v / $INTERVAL; + } + if ( my $l = $last->{$instance}->{nic}->[$nic] ) { + $stat->{$instance}->{nic}->[$nic]->{$_} = $d->{$_} - $l->{$_} foreach keys %$d; + $stat->{$instance}->{nic}->[$nic]->{dev} = $dev; + if ( -e "/sys/class/net/$dev/master" ) { + my $vlan = readlink "/sys/class/net/$dev/master"; + $vlan =~ s/^.*br//; + $stat->{$instance}->{nic}->[$nic]->{vlan} = $vlan; + } } + $last->{$instance}->{nic}->[$nic] = $d; + + } +} + +#XXX $stat; +#XXX $last; + +sub dump4influx { + my $hash = shift; + my @v; + foreach my $k ( keys %$hash ) { + my $v = $hash->{$k}; + my ( $d, $s ) = $v =~ m/^\d+(\.\d+)?$/i ? ( '', '' ) : # float + ( '"' ,'' ) ; # string + + push @v, "$k=$d$v$d$s"; + } + my $i = join(',', @v); + return $i; +} + +open(my $fh, '>', '/dev/shm/gnt-monitor.influx'); + +foreach my $instance ( keys %$stat ) { + + next if $skip->{$instance}; + + print $fh qq{ganeti_cpu,node=$node,instance=$instance$location cpu=$stat->{$instance}->{cpu},ticks=$stat->{$instance}->{ticks} $t\n}; + + foreach my $disk ( @{ $stat->{$instance}->{disk} } ) { + print $fh qq{ganeti_disk,node=$node,instance=$instance$location },dump4influx( $disk ), " $t\n"; + } + + foreach my $nic ( @{ $stat->{$instance}->{nic} } ) { + my $vlan = delete $nic->{vlan}; + $vlan = 0 unless defined $vlan; + print $fh qq{ganeti_nic,node=$node,instance=$instance,vlan=${vlan}$location },dump4influx( $nic ), " $t\n"; } + +} + +close($fh); + +if ( system( 'curl', '-s', '-XPOST', $INFLUX, '--data-binary', '@/dev/shm/gnt-monitor.influx' ) == 0 ) { +} else { + rename '/dev/shm/gnt-monitor.influx', '/dev/shm/gnt-monitor.influx.bug.' . scalar glob '/dev/shm/gnt-monitor.influx.bug.*'; + warn "curl failed: $? $!"; } -XXX $stat; +my $dt = time() - $begin_t; +sleep $INTERVAL - $dt if $dt > 0 && $dt < $INTERVAL; +} #/while