From: Dobrica Pavlinusic <dpavlin@rot13.org>
Date: Sat, 8 Oct 2011 16:31:49 +0000 (+0200)
Subject: split md5sum db per user
X-Git-Url: http://git.rot13.org/?a=commitdiff_plain;h=f4e96fe499a96f2cac280df03a6336f869edcd65;p=cloudstore.git

split md5sum db per user
---

diff --git a/README b/README
index 9a70256..259b52a 100644
--- a/README
+++ b/README
@@ -2,7 +2,7 @@ Central rsync backup server
 
 Features:
 
-RSYNC_PORT=6502 ./rsync-piper.pl
+sudo ./rsync-piper.pl
 	- generate rsync configuration based on content of RSYNC_DIR/users
 	- start rsync server on specified port
 	- follow rsync transfer log, parse it and trigger events
@@ -11,9 +11,7 @@ RSYNC_PORT=6502 ./rsync-piper.pl
 	- gearman log shipping in json format
 	- support login+host usernames for client secrets
 	- push md5sum file to create files from server's md5 pool
-
-data-worker.pl
-	- gearman worker to store rsync transfer log into PostgreSQL
+	- track quota usage using uid/gid using libnss-extrausers
 
 create-user.sh
 remove-user.sh
diff --git a/dump-bdb.pl b/dump-bdb.pl
index bfe9884..efad525 100755
--- a/dump-bdb.pl
+++ b/dump-bdb.pl
@@ -5,7 +5,10 @@ use strict;
 use BerkeleyDB;
 use Data::Dump qw(dump);
 
+my $path = shift @ARGV;
+die $! unless -r $path;
+
 my %h;
-tie %h, "BerkeleyDB::Hash", -Filename => $_[0];
+tie %h, "BerkeleyDB::Hash", -Filename => $path;
 
 print dump( \%h );
diff --git a/lib/CloudStore/Store.pm b/lib/CloudStore/Store.pm
index 4cb8e18..f49785b 100644
--- a/lib/CloudStore/Store.pm
+++ b/lib/CloudStore/Store.pm
@@ -106,8 +106,10 @@ sub new_link {
 #				});
 #				$self->new_file($origin);
 				warn "INFO: sent file ",dump($l,$f);
-#				my $md5 = $self->{md5}->{$s} || die "no md5 for $s";
-#				$self->{md5}->{$d} = $md5;
+				my $md5sum = $self->md5sum($data);
+
+				my $md5 = $md5sum->{$s} || die "no md5 for $s";
+				$md5sum->{$d} = $md5; # FIXME broken!
 			}
 
 
@@ -117,20 +119,39 @@ sub new_link {
 	}
 }
 
+our $md5_login;
+sub md5sum {
+	my ( $self, $data ) = @_;
+
+	if ( exists $md5_login->{$data->{login}} ) {
+		return $md5_login->{$data->{login}};
+	} elsif ( my $login = $data->{login} ) {
+
+		my $md5_path = $self->{dir} || die "no dir?";
+		$login =~ s/^u//;
+		$md5_path .= "/$login/.md5.db";
+
+		my %md5;
+		my $db = tie %md5, 'BerkeleyDB::Hash',
+			-Filename => $md5_path,
+			-Flags => DB_CREATE,
+		;
+
+		return $md5_login->{$login} = \%md5;
+	} else {
+		confess "can't open md5sum";
+	}
+}
+
 sub rsync_log {
 	my ( $self, $data ) = @_;
 	if ( $data =~ m/\[(\d+)\] rsync \w+ (\S+) \w+ (\S+)/ ) {
 		my ( $pid, $module, $login ) = ( $1, $2, $3 );
+
 		$login =~ s/\@.+//;
 		my ( undef, undef, $uid, $gid, undef, undef, $email, $dir, $shell ) =
 			getpwnam $login;
 
-		my %md5;
-		my $db = tie %md5, 'BerkeleyDB::Hash',
-			-Filename => "$dir/.md5.db",
-			-Flags => DB_CREATE
-		;
-
 		$self->{pid}->{$pid} = {
 			login => $login,
 			uid => $uid,
@@ -138,16 +159,17 @@ sub rsync_log {
 			email => $email,
 			dir => $dir,
 			shell => $shell,
-			md5 => \%md5,
 		};
 
-	} elsif ( $data =~ m/\[(\d+)\] sent \d+ bytes\s+received \d+ bytes/ ) {
+		warn "created $pid";
+
+	} elsif ( $data =~ m/\[(\d+)\] sent \S+ bytes\s+received \S+ bytes/ ) {
 		my $pid = $1;
-		untie $self->{pid}->{$pid}->{md5} && warn "md5 hash untied";
-		warn "removed ", dump
+		untie $md5_login->{ $self->{$pid}->{login} } && warn "untie $pid";
 		delete $self->{pid}->{$pid};
+		warn "removed $pid";
 	} else {
-		warn "## rsync_log $data";
+#		warn "## rsync_log $data";
 	}
 }
 
@@ -203,8 +225,8 @@ sub md5pool {
 		link $path, "$pool/$md5";
 	}
 
-	my $md5hash = $self->{pid}->{ $data->{pid} }->{md5} || die "no md5 for $data->{pid} in ",dump( $self->{pid} );
-	$md5hash->{ $data->{file} } = $md5;
+	my $md5sum = $self->md5sum($data);
+	$md5sum->{ $data->{file} } = $md5;
 }
 
 my $empty_md5 = " " x 32;
diff --git a/test.sh b/test.sh
index 8b6504c..70f1b38 100755
--- a/test.sh
+++ b/test.sh
@@ -20,7 +20,7 @@ pull() {
 sudo ./remove-user.sh test@example.com
 sudo ./remove-user.sh test2@example.com
 
-RSYNC_LOGIN=`sudo ./create-user.sh test@example.com secret 200k`
+RSYNC_LOGIN=`sudo ./create-user.sh test@example.com secret 500k`
 RSYNC_LOGIN2=`sudo ./create-user.sh test2@example.com secret 20k`
 
 dir=/tmp/test
@@ -69,6 +69,8 @@ md5sum clone/* | sed -e 's!  !  _import/foo/bar/baz!' > md5sum
 cat md5sum
 push md5sum -v
 
+exit
+
 # test send
 
 ps ax >> sent-to-$RSYNC_LOGIN2