X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;f=rsync-piper.pl;h=04c3a8a6b44be031cb43885563a40898e8f99d80;hb=41187c9904153b484a817883805fb8ad6a206b40;hp=dc7333eefc7b9e8c7a55f3ffc0e75df3a23c0173;hpb=8d7f0aed45b97b698cd616fcd3986153be1316a7;p=cloudstore.git diff --git a/rsync-piper.pl b/rsync-piper.pl index dc7333e..04c3a8a 100755 --- a/rsync-piper.pl +++ b/rsync-piper.pl @@ -12,15 +12,16 @@ use English; use Module::Refresh; use lib 'lib'; -use CloudStore::Couchbase; +use WarnColor; +use CloudStore::Store; -my $dir = $ENV{RSYNC_DIR} || '/srv/cloudstore'; -my $port = $ENV{RSYNC_PORT} || 6501; -my $users = "users"; +my $slice = $ARGV[0] || 's1'; -my $log_fifo = "$dir/var/$port.log"; -my $pid_file = "$dir/var/$port.pid"; -my $cfg_file = "$dir/var/$port.conf"; +my ( undef, $dir, $port, undef ) = getgrnam($slice); + +my $log_fifo = "$dir/$port.log"; +my $pid_file = "$dir/$port.pid"; +my $cfg_file = "$dir/$port.conf"; my $rsync = 'rsync'; $rsync = 'bin/rsync' if -x 'bin/rsync'; # use 3.1dev version! @@ -54,13 +55,13 @@ if ( $ENV{SQL} ) { exit 1; } -my $store = CloudStore::Couchbase->new; - -mkdir "$dir/var" if ! -e "$dir/var"; +my $store = CloudStore::Store->new( $slice ); unlink $log_fifo if -f $log_fifo; mkfifo $log_fifo, 0700 unless -p $log_fifo; +sub rsync_rebuild_config { + my $rsync_config = qq{ #uid = nobody @@ -69,7 +70,7 @@ my $rsync_config = qq{ use chroot = no #max connections = 4 -lock file = $dir/var/$port.lock +lock file = $dir/$port.lock #syslog facility = local5 log file = $log_fifo @@ -83,44 +84,49 @@ pid file = $pid_file # don't check secrets file permission (uid) strict modes = no -pre-xfer exec = /srv/cloudstore/rsync-xfer-trigger.pl -post-xfer exec = /srv/cloudstore/rsync-xfer-trigger.pl +#pre-xfer exec = /srv/cloudstore/rsync-xfer-trigger.pl +#post-xfer exec = /srv/cloudstore/rsync-xfer-trigger.pl + +# inplace breaks update of deduped files +refuse options = inplace }; -foreach my $path ( glob "$users/*" ) { +open(my $p, '<', '/var/lib/extrausers/passwd'); +while(<$p>) { + chomp; - my $login = $path; - $login =~ s{^.+/([^/]+)$}{$1}; + my ( $login, undef, $uid, $gid, $email, $path, $shell ) = split(/:/,$_,7); - if ( -d $path && -d "$path/blob" && -f "$path/secrets" ) { - my @secrets = map { chomp; $_ } read_file "$path/secrets"; + if ( -d $path && -f "$path/.secrets" && ! -e "$path/.meta/secrets" ) { + $store->api->mkbasepath( "$path/.meta/secrets" ); + rename "$path/.secrets", "$path/.meta/secrets"; + warn "UPGRADE $login rsync secrets location\n"; + } + + if ( -d $path && -f "$path/.meta/secrets" ) { + my @secrets = map { chomp; $_ } read_file "$path/.meta/secrets"; my $auth_users = join(', ', map { s/:.+$//; $_ } @secrets ); $rsync_config .= <<__RSYNC_MODULE__; [$login] - path = $dir/users/$login/blob + path = $path auth users = $auth_users - secrets file = $dir/users/$login/secrets + secrets file = $path/.meta/secrets read only = false + uid = $uid + gid = $gid + filter = - /.meta +# refuse options = c delete +# dont compress = * + incoming chmod = u=rwX,g+rX,o+rX + __RSYNC_MODULE__ print "INFO: added $login = $auth_users\n"; - my $quota = read_file "$path/quota" if -e "$path/quota"; - chomp $quota && $quota * 1; - $quota ||= 100 * 1024; # Kb - XXX default quota - - $store->user_set({ - login => $login, - path => $path, - secrets => [ @secrets ], - quota => $quota, - port => $port, - }); - } else { warn "skipped $login: $!"; } @@ -130,6 +136,10 @@ __RSYNC_MODULE__ write_file $cfg_file, $rsync_config; warn "created $cfg_file ", -s $cfg_file, " bytes\n"; +} # sub rsync_rebuild_config + +rsync_rebuild_config; + sub rsync_running_pid { return unless -e $pid_file; my $pid = read_file $pid_file; @@ -140,6 +150,8 @@ sub rsync_running_pid { if ( my $pid = rsync_running_pid ) { if ( kill 0, $pid ) { warn "found rsync pid $pid\n"; + kill 1, $pid && warn "reload config"; +=for kill-rsync kill 2, $pid; while ( -e $pid_file ) { warn "waiting for rsync to die...\n"; @@ -153,6 +165,7 @@ if ( my $pid = rsync_running_pid ) { } kill 0, $pid && die "can't kill it!"; +=cut } else { unlink $pid_file; } @@ -170,7 +183,7 @@ sub REAPER { $SIG{CHLD} = \&REAPER; -if ( ! -e $pid_file ) { +if ( ! -e $pid_file || ! kill( 0, rsync_running_pid ) ) { my $exec = "$rsync --daemon --config $cfg_file --no-detach --port=$port"; warn "START $exec\n"; @@ -194,19 +207,21 @@ $gearman->job_servers('127.0.0.1:4730'); while(1) { die "no rsync running" unless kill 0, rsync_running_pid; - warn "# reading log output from $log_fifo\n"; + warn "waiting for log from $log_fifo\n"; open(my $fifo, '<', $log_fifo); while( my $line = <$fifo> ) { Module::Refresh->refresh; chomp $line; - print $line, $/; + warn $line, $/; - if ( $line =~ /transfer-log:(.+)/ ) { - my $transfer = $1; + if ( $line =~ /\[(\d+)\] transfer-log:(.+)/ ) { + my $pid = $1; + my $transfer = $2; $transfer =~ s|(\d\d\d\d)/(\d\d)/(\d\d)[-\s](\d\d:\d\d:\d\d)|$1-$2-$3T$4|g; my ( $yyyy,$mm,$dd,undef,$login,undef ) = split( /[\-T\|]/, $transfer, 6 ); my $host = $1 if $login =~ s/\+(.+)//; +if(0) { my $path = "users/$login/log"; mkdir $path unless -d $path; $path .= "/$yyyy-$mm-$dd"; @@ -215,19 +230,31 @@ while(1) { print $log join('|',@transfer),"\n" if $new_log; # store header print $log "$transfer\n"; close $log; +} my @v = split(/\|/,$transfer,$#transfer + 1); my %data; @data{@transfer_names} = @v ; # FIXME validate? + $data{pid} = $pid; + # overwrite pid from transfer log with consistant one for start/stop + print ">>> data ",dump( \%data ) if $ENV{DEBUG}; - $store->transfer( \%data ); + $store->rsync_transfer( \%data ); =for gearman $gearman->dispatch_background( 'rsync_transfer' => $json ); =cut die "no rsync running" unless kill 0, rsync_running_pid; + } elsif ( $line =~ m/(unknown module|rebuild|reload|config)/ ) { + warn "refresh modules, rebuild config and HUP rsync"; + Module::Refresh->refresh; + rsync_rebuild_config; + my $pid = rsync_running_pid; + kill 1, $pid && warn "reload config"; + } else { + $store->rsync_log( $line ); } } close($fifo);