create user/log/yyyy-mm-dd transfer logs

diff --git a/rsync-piper.pl b/rsync-piper.pl
index 20f27d4..27d2f22 100755 (executable)
--- a/rsync-piper.pl
+++ b/rsync-piper.pl
@@ -14,29 +14,29 @@ use JSON::XS;
 my $dir = '/srv/cloudstore/var';
 my $log_fifo = "$dir/rsyncd.log";
 my $pid_file = "$dir/rsyncd.pid";
-my $cfg_file   = "$dir/rsyncd.conf";
+my $cfg_file = "$dir/rsyncd.conf";
 my $users    = "users";
 
+my @transfer = qw(
+timestamp:%t:timestamp
+login:%u:text
+pid:%p:int
+perms:%B:text
+itemize:%i:text
+mtime:%M:timestamp
+op:%o:text
+size:%l:int
+transfered:%b:int
+file:%f:text
+);
+
+my @transfer_names =          map { ( split(/:/,$_,3) )[0] } @transfer;
+my $transfer_log   = join(' ',map { ( split(/:/,$_,3) )[1] } @transfer );
+
 mkdir $dir if ! -e $dir;
 
 mkfifo $log_fifo, 0700 unless -p $log_fifo;
 
-my $transfer_log = {
-       ip => '%a',
-       login => '%u',
-       host => '%h',
-       perms => '%B',
-       file => '%f',
-       updated => '%i',
-       len => '%l',
-       transfered => '%b',
-#      module => '%m',
-       mtime => '%M',
-       op => '%o',
-       pid => '%p',
-       timestamp => '%t',
-};
-
 my $rsync_config = qq{
 
 #uid = nobody
@@ -51,7 +51,7 @@ lock file = $dir/rsyncd.lock
 log file  = $log_fifo
 
 transfer logging = yes
-log format = transfer-log:} . join('|',values %$transfer_log) . qq{
+log format = transfer-log:$transfer_log
 max verbosity = 5
 
 pid file  = $pid_file
@@ -99,41 +99,44 @@ if ( ! -e $pid_file ) {
        }
 }
 
-
 use Gearman::Client;
-my $client = Gearman::Client->new;
-$client->job_servers('127.0.0.1:4730');
+my $gearman = Gearman::Client->new;
+$gearman->job_servers('127.0.0.1:4730');
 
 while(1) {
        warn "# reading log output from $log_fifo\n";
-       open(my $log, '<', $log_fifo);
-       while( my $line = <$log> ) {
-               print "LINE: $line";
-               if ( $line =~ /transfer-log:(.+\|.+)/ ) {
+       open(my $fifo, '<', $log_fifo);
+       while( my $line = <$fifo> ) {
+               chomp $line;
+               print $line, $/;
+               if ( $line =~ /transfer-log:(.+)/ ) {
+                       my $transfer = $1;
+                       $transfer =~ s|(\d\d\d\d)/(\d\d)/(\d\d)[-\s](\d\d:\d\d:\d\d)|$1-$2-$3T$4|g;
+warn "XXX $transfer";
+                       my ( $yyyy,$mm,$dd,undef,$login,undef ) = split( /[\-T\s]/, $transfer, 6 );
+
+                       my $path = "users/$login/log";
+                       mkdir $path unless -d $path;
+                       $path .= "/$yyyy-$mm-$dd";
+warn "## $path $transfer\n";
+                       my $new_log = ! -e $path;
+                       open( my $log, '>>', $path );
+                       print $log join(' ',@transfer),"\n" if $new_log; # store header
+                       print $log "$transfer\n";
+                       close $log;
+
+
+                       my @v = split(/\s+/,$transfer,$#transfer + 1);
                        my %data;
-                       my @k = keys %$transfer_log;
-                       my @v = split(/\|/,$1);
-                       @data{@k} = @v ; # FIXME validate?
-
-                       # fixup data
-                       $data{mtime} =~ s|^(\d\d\d\d)/(\d\d)/(\d\d)-(\d\d:\d\d:\d\d)|$1-$2-$3 $4| && warn "fixed mtime ISO $data{mtime}\n";
-
-                       print "transfer-log:",dump(\%data),$/;
-
-                       my $json = encode_json \%data;
+                       @data{@transfer_names} = @v ; # FIXME validate?
 
-                       my $path = sprintf "users/%s/data/%.5f-%d",
-                               $data{login}, Time::HiRes::time(), $data{pid};
-                       open(my $fh, '>', $path);
-                       print $fh $json;
-                       close $fh;
-                       print $path, " ", -s $path, " bytes\n";
+                       print ">>> data ",dump( \%data );
 
-                       $client->dispatch_background( 'log' => $json );
+                       $gearman->dispatch_background( 'rsync_transfer' => encode_json \%data );
 
                }
        }
-       close($log);
+       close($fifo);
        sleep 1;
 }