Merge branch 'master' of git.rot13.org:/git/cloudstore

diff --git a/client/rsync-push.sh b/client/rsync-push.sh
index 62f0818..cf966c7 100755 (executable)
--- a/client/rsync-push.sh
+++ b/client/rsync-push.sh
@@ -2,5 +2,5 @@
 
 #inotifywait -m -r --format '%:e %f' Dropbox/
 pwd > .pwd
-RSYNC_PASSWORD=secret rsync .pwd "$@" rsync://dpavlin@10.60.0.244:6501/dpavlin/`hostname`/
+RSYNC_PASSWORD=secret rsync .pwd "$@" rsync://$USER@10.60.0.244:6501/$USER/`hostname`/
 rm .pwd

diff --git a/lib/CloudStore/Couchbase.pm b/lib/CloudStore/Couchbase.pm
index ddd5c8b..c59a1ac 100644 (file)
--- a/lib/CloudStore/Couchbase.pm
+++ b/lib/CloudStore/Couchbase.pm
@@ -12,6 +12,8 @@ use Data::Dump qw(dump);
 use LWP::Simple;
 use Carp qw(confess);
 
+use WarnColor;
+
 my $buckets = {
        users => 5800,
        files => 5801,
@@ -28,7 +30,7 @@ sub new {
                my $port = $buckets->{$bucket};
                my $server = new Cache::Memcached {
                        'servers' => [ "127.0.0.1:$port" ],
-                       'debug' => $ENV{DEBUG},
+                       'debug' => defined $ENV{DEBUG} && $ENV{DEBUG} > 3,
                #       'compress_threshold' => 10_000,
                };
                #$server->set_servers($array_ref);
@@ -48,14 +50,14 @@ sub json_set {
        confess "data not ref ",dump($data) unless ref $data;
        my $json = encode_json $data;
        $self->{$bucket}->set( $key => $json );
-       warn "## $bucket set $key $json\n";
+       warn "## json_set $bucket $key $json\n";
        return $json;
 }
 
 sub json_get {
        my ($self,$bucket,$key,$data) = @_;
        if ( my $json = $self->{$bucket}->get($key) ) {
-               warn "## $bucket get $key $json\n";
+               warn "## json_get $bucket $key $json\n";
                return decode_json $json;
        }
 }
@@ -71,7 +73,7 @@ sub user_get {
        my $user = $self->json_get( 'users', $login );
        $user->{usage} = $self->usage( $login );
        $user->{status} = $self->status( $login );
-       warn "## user ",dump($user);
+       warn "## user ",dump($user) if $ENV{DEBUG};
        return $user;
 }
 
@@ -80,6 +82,7 @@ sub status {
        $login = $login->{login} if ref $login;
        if ( $message ) {
                $self->{session}->set( "$login:status" => $message );
+               return $message;
        } else {
                $self->{session}->get( "$login:status" );
        }
@@ -185,24 +188,50 @@ sub new_link {
        if ( $data->{file} =~ m{^(.*/?)\.send/([^/]+)/(.+)$} ) {
                my ( $dir, $to, $name ) = ( $1, $2, $3 );
                my $path = "users/$data->{login}/blob/" . $data->{file};
-               my $file = readlink $path;
-               warn "## $path -> $file";
-               if ( $file =~ s{^\Q/rsyncd-munged/../../\E}{$dir} ) {
-                       warn "SEND To:$to Name:$name File:$file\n";
-                       my $s = "users/$data->{login}/blob/$file";
+               my $link_to = readlink $path;
+               warn "$link_to";
+               if ( $link_to =~ s{^\Q/rsyncd-munged/\E}{/} ) {
+
+                       my $s = $path;
+                       $s =~ s{/[^/]+$}{}; # strip filename
+                       while ( $link_to =~ s{/../}{/} ) {
+                               $s =~ s{/[^/]+$}{} || die "can't strip $s";
+                               warn "## simplify $s $link_to\n";
+                       }
+                       $s .= $link_to;
+
                        my $d = "users/$to/blob";
-                       die "no user $to" unless -e $d;
+                       if ( ! -e $d ) {
+                               warn "ERROR: no to user $to in $d";
+                               return;
+                       }
                        $d .= "/$name";
 
-                       # since $name can contain directories we must create them
+                       # $name can contain directories so we must create them
                        my $to_dir = $d;
                        $to_dir =~ s{/[^/]+$}{};
                        make_path $to_dir if ! -e $to_dir;
 
-                       warn "link $s -> $d\n";
-                       link $s, $d;
+                       if ( ! -e $s ) {
+                               warn "ERROR: can't find source $s";
+                       } else {
+
+                               warn "link $s -> $d\n";
+                               link $s, $d;
+
+                               my ($l,$f) = ($1,$2) if $s =~ m{users/([^/]+)/blob/(.+)};
+
+                               my $origin = $self->file_get({
+                                       login => $l,
+                                       file  => $f,
+                               });
+                               $self->new_file($origin);
+                               warn "INFO: sent file ",dump($origin);
+                       }
+
+
                } else {
-                       warn "ERROR: can't SEND To:$to Name:$name File:$file";
+                       warn "ERROR: can't SEND To:$to Name:$name Link:$link_to";
                }
        }
 }
@@ -280,7 +309,7 @@ sub dedup {
                                        size => -s $new,
                                };
                                $self->new_file($fake);
-                               warn "fake ",dump($fake);
+                               warn "import from $path ",dump($fake);
                        } else {
                                md5pool $new => $md5;
                        }

diff --git a/lib/WarnColor.pm b/lib/WarnColor.pm
new file mode 100644 (file)
index 0000000..5cf738d
--- /dev/null
+++ b/lib/WarnColor.pm
@@ -0,0 +1,31 @@
+package WarnColor;
+
+use warnings;
+use strict;
+
+sub BEGIN {
+
+       sub port2color {
+               my $port = shift;
+               return "\e[1m0\e[0m" if $port == 0;
+
+               my $c = ( $port % 6 ) + 31;
+               return "\e[${c}m$port\e[0m";
+       }
+
+       $SIG{__WARN__} = sub {
+               return unless @_;
+               my $msg = join('', @_);
+               if ( $msg =~ s{ line (\d+)\.}{ +$1} ) {
+                       $msg =~ s{^(.+)( at .+)}{\e[31m$1\e[0m$2} if $msg !~ m{^#};
+               }
+               $msg =~ s{\[(\d+)\]}{ '[' . port2color($1) . ']' }eg;
+               $msg =~ s{(\||=>)}{\e[34m$1\e[0m}g; # blue
+               $msg =~ s{(["\{\}\#])}{\e[33m$1\e[0m}g; # yellow
+               print STDERR $msg unless $msg =~ m{^#} && ! $ENV{DEBUG};
+               return 1;
+       };
+
+}
+
+1;

diff --git a/rsync-piper.pl b/rsync-piper.pl
index 72a55fc..1aaca5b 100755 (executable)
--- a/rsync-piper.pl
+++ b/rsync-piper.pl
@@ -12,6 +12,7 @@ use English;
 use Module::Refresh;
 
 use lib 'lib';
+use WarnColor;
 use CloudStore::Couchbase;
 
 my $dir   = $ENV{RSYNC_DIR}  || '/srv/cloudstore';
@@ -196,12 +197,12 @@ $gearman->job_servers('127.0.0.1:4730');
 
 while(1) {
        die "no rsync running" unless kill 0, rsync_running_pid;
-       warn "# reading log output from $log_fifo\n";
+       warn "waiting for log from $log_fifo\n";
        open(my $fifo, '<', $log_fifo);
        while( my $line = <$fifo> ) {
                Module::Refresh->refresh;
                chomp $line;
-               print $line, $/;
+               warn $line, $/;
 
                if ( $line =~ /transfer-log:(.+)/ ) {
                        my $transfer = $1;

diff --git a/rsync-xfer-trigger.pl b/rsync-xfer-trigger.pl
index a6478f5..97018e4 100755 (executable)
--- a/rsync-xfer-trigger.pl
+++ b/rsync-xfer-trigger.pl
@@ -15,7 +15,7 @@ my $user  = $store->user_get($login);
 
 my $ok = $user->{usage} <= $user->{quota};
 
-warn $store->status( $login => sprintf 'usage: %s %d %s %d'
+print "$login ", $store->status( $login => sprintf 'quota: %s %d %s %d'
        , ( $ok ? 'OK' : 'ERROR' )
        , $user->{usage}
        , ( $ok ? '<=' : '>' )
@@ -23,5 +23,5 @@ warn $store->status( $login => sprintf 'usage: %s %d %s %d'
 ),$/;
 
 my $exit = $ok ? 0 : 1;
-warn "# exit $exit";
+warn "# exit $exit" if $ENV{DEBUG};
 exit $exit;

diff --git a/test.sh b/test.sh
index dafa4a6..5667336 100755 (executable)
--- a/test.sh
+++ b/test.sh
@@ -5,13 +5,13 @@ rm -Rvf $user_dir/log/* #$user_dir/blob/*
 
 push() {
        file=`shift`
-       RSYNC_PASSWORD=secret rsync $* $file rsync://test+localhost@127.0.0.1:6501/test/$file
+       RSYNC_PASSWORD=secret rsync $* $file rsync://test+localhost@127.0.0.1:6501/test/localhost/$file
 }
 pull() {
        from=$1
        to=$2
        test -z "$to" && to=$1
-       RSYNC_PASSWORD=secret rsync rsync://test+localhost@127.0.0.1:6501/test/$from $to
+       RSYNC_PASSWORD=secret rsync rsync://test+localhost@127.0.0.1:6501/test/localhost/$from $to
 }
 
 
@@ -19,6 +19,10 @@ pull() {
 
 ./create-user.sh test secret
 ./create-user.sh test secret localhost
+./store-fsck.pl test
+
+./remove-user.sh test2
+./create-user.sh test2 secret
 
 dir=/tmp/test
 rm -Rf "$dir"
@@ -66,6 +70,16 @@ md5sum clone/* | sed -e 's!  !  _import/foo/bar/baz!' > md5sum
 cat md5sum
 push md5sum -v
 
+# test send
+
+ps ax >> sent-to-test2
+push sent-to-test2 # must be done before first send
+
+mkdir -p .send/test2/received
+ln -s ../../../sent-to-test2 .send/test2/received/new-name
+
+push .send -rvl
+
 tail -20 $user_dir/log/*
 
 #diff -urw $user_dir/blob $dir/