X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;f=lib%2FCloudStore%2FAPI.pm;h=1b2625d9199fbee886210d609a5876d891d91233;hb=3b458e4db1acf5a7f200f38d629bed9f38632abd;hp=0f709c9662ac0bea69ae11fbb2e71b9256b6f75c;hpb=bfab7a952b5a81832822807b0bdb818af19924cd;p=cloudstore.git diff --git a/lib/CloudStore/API.pm b/lib/CloudStore/API.pm index 0f709c9..1b2625d 100644 --- a/lib/CloudStore/API.pm +++ b/lib/CloudStore/API.pm @@ -9,35 +9,47 @@ use base qw(CloudStore::Gearman CloudStore::MD5sum); use File::Path qw(make_path remove_tree); use File::Find; use Data::Dump qw(dump); -use Carp qw(confess); +use Carp qw(confess cluck); sub new { my ($class,$slice) = @_; - my ( undef, $dir, $port, undef ) = getgrnam($slice); - die "can't find group $slice: $!" unless $dir && $port; + cluck "DEPRICIATED $slice specified" if $slice; + my $self = { passwd => '/var/lib/extrausers/passwd', - PORT => $port, - SLICE => $dir, }; bless $self, $class; $self->{md5} = $self->user_info("md5") || die "can't find user md5"; +=for removed $self->{md5}->{dir} = "$dir/md5"; if ( ! -e $self->{md5}->{dir} ) { make_path $self->{md5}->{dir}, { uid => $self->{md5}->{uid}, gid => $self->{md5}->{gid} }; warn "## CREATED md5pool $self->{md5}->{dir}\n"; } - - my $name = $self->{SLICE}; - $name =~ s/\W+/_/g; - $name =~ s/^_+//; - $self->{quota} = $name . '_quota'; +=cut return $self; } +sub slice_dir_port { + my ($self,$slice) = @_; + my ( undef, $dir, $port, undef ) = getgrnam($slice); + die "getgrnam $slice: $!" if $!; + warn "# slice_dir_port $slice = $dir $port\n"; + return ( $dir, $port ); +} + +sub dir2gearman { + my $self = shift; + my $dir = shift; + $dir =~ s/\W+/_/g; + $dir =~ s/^_+//; + $dir =~ s{_\d+$}{}; + return join('_', $dir, @_); +} + sub user_info { my ($self,$login) = @_; @@ -61,21 +73,28 @@ sub create_user { while(<$fh>) { my ( $login, $passwd, $uid, $gid, $email, $dir, $shell ) = split(/:/,$_); $max_uid = $uid if $uid > $max_uid; - $found = $uid if $email eq $new_email; + $found = $login if $email eq $new_email; } close($fh); + my $slice = $ENV{SLICE} || 's1'; + $slice =~ s{/.+/(\w+)$}{$1}; + my ( $dir, $port ) = $self->slice_dir_port( $slice ); + + $dir ||= $ENV{SLICE}; + $port ||= 6501; + if ( ! $found ) { $max_uid++; - my $dir = "$self->{SLICE}/$max_uid"; - warn "# create_user $new_email $new_quota = $max_uid $dir"; + $dir .= "/$max_uid"; + warn "# create_user $slice $new_email $new_quota = $max_uid $dir"; open(my $fh, '>>', $self->{passwd}); - print $fh "u$max_uid:$new_passwd:$max_uid:$self->{PORT}:$new_email:$dir:/bin/true\n"; + print $fh "u$max_uid:$new_passwd:$max_uid:$port:$new_email:$dir:/bin/true\n"; close($fh); - $found = $max_uid; + $found = "u$max_uid"; mkdir $dir; - chown $max_uid, $self->{PORT}, $dir; + chown $max_uid, $port, $dir; my $path = "$dir/.meta/secrets"; $self->mkbasepath($path); @@ -85,19 +104,26 @@ sub create_user { } # FIXME update quota only on create? - $self->gearman_do( "$self->{quota}_set" => "$found $new_quota" ); + $self->gearman_do( $self->dir2gearman( $dir, 'quota', 'set' ) => "$found $new_quota" ); return $found; } sub mkbasepath { my ($self,$path,$opts) = @_; + cluck "ERROR: mkbasepath called without opts, so user is root!" unless $opts; + if ( $ENV{DEBUG} ) { + warn "# mkbasepath $path ",dump($opts); + $opts->{verbose} ||= 1; + } $path =~ s{/[^/]+$}{}; - make_path $path unless -d $path; + if ( ! -d $path ) { + make_path $path, $opts; + } } sub user_dir { - my ( $self,$user, $dir ) = @_; + my ( $self, $user, $dir ) = @_; $user = $self->user_info($user) unless ref $user eq 'HASH'; my $path; if ( exists $user->{dir} ) { @@ -141,6 +167,23 @@ sub append_meta { sub usage { my ( $self, $user ) = @_; $user = $self->user_info($user) unless ref $user eq 'HASH'; + + my $usage_path = $user->{dir} . '/.meta/files.usage'; + $self->mkbasepath( $usage_path, { uid => $user->{uid} } ); + if ( ! -e $usage_path ) { + warn "# usage $usage_path missing"; + $self->list_files($user); + } + + open(my $fh, '<', $usage_path); + my $size = <$fh>; + chomp $size; + + warn "# usage $user->{login} $size bytes\n"; + return $size; + +=for slow and broken + my $path = $self->user_dir( $user => 'usage'); my $sum; open(my $fh, '<', $path); @@ -151,19 +194,22 @@ sub usage { $sum->{_usage} += $v[1]; } my ( $usage, $quota ) = split(/ /, - $self->gearman_do( "$self->{quota}_get" => $user->{uid} ) + $self->gearman_do( $self->dir2gearman( $user->{dir}, 'quota', 'get' ) => $user->{uid} ) ); $sum->{_usage} += $usage; $sum->{_quota} = $quota; warn "## usage ",dump($user, $sum), $/; return $sum; + +=cut + } sub send_file { my ( $self, $f_uid,$f_path, $t_uid,$t_path ) = @_; my $f = $self->user_info($f_uid); - die "FIXME not on current slice" if $f->{dir} !~ m/^$self->{SLICE}/; + die "FIXME not on current slice" if $f->{dir} !~ m/^$ENV{SLICE}/; my $t = $self->user_info($t_uid); my $f_full = "$f->{dir}/$f_path"; @@ -207,6 +253,9 @@ sub send_file { if ( $ok ) { $self->append( $t, 'recv', $size, $f->{uid}, $t_path ); $self->append_meta('md5sum', $t, $md5 => $t_path ) if $md5; # md5sum for received files! FIXME -- cross-slice md5 + $self->refresh_file_list( $t ); + } else { + warn "ERROR: send_file $f_full -> $t_full: $!"; } return $size; @@ -222,6 +271,8 @@ sub rename_file { $self->mkbasepath( $t_full, { uid => $user->{uid}, gid => $user->{gid} } ); my $ok = rename $f_full, $t_full; + $self->refresh_file_list( $user ); + my $md5 = $self->md5_get($t_full); if ( ! $md5 ) { warn "ERROR: no md5sum for $from"; @@ -266,6 +317,8 @@ sub delete { $self->append( $user, 'delete', -$deleted_size, $user->{uid}, $path ); $self->append_meta('md5sum', $user, 'delete', $path ); + $self->refresh_file_list( $user ); + return $full_path; } @@ -274,7 +327,72 @@ sub file_size { $user = $self->user_info($user) unless ref $user eq 'HASH'; my $full_path = "$user->{dir}/$path"; - return -s $full_path; + my $size = -s $full_path; + warn "# file_size $full_path = $size bytes\n"; + return $size; +} + +sub list_files { + my ( $self, $user, $path ) = @_; + + $user =~ s{/+$}{} && warn "cleanup list_files arg [$user]"; + + $user = $self->user_info($user) unless ref $user eq 'HASH'; + + die "no dir for ",dump($user) unless exists $user->{dir}; + + my $files = $user->{dir} . '/.meta/files'; + $self->mkbasepath( $files, { uid => $user->{uid} } ); + if ( -e $files && -s $files > 0 && -e "$files.usage") { + local $/ = undef; + open(my $fh, '<', $files); + my $list = <$fh>; + close($fh); + warn "# list_files $user->{login} from cache ", length($list), " bytes\n"; + return $list; + } + + my $dir = $user->{dir}; + open(my $pipe, '-|', qq|find -L $dir -printf "%y %s %p\n"|); + open(my $fh, '>', "$files.new"); + my $total_usage = 0; + my $list_txt; + while(<$pipe>) { + chomp; + my ( $type, $size, $name ) = split(/\s/, $_, 3); + $name =~ s{$dir}{./}; + $name =~ s{//+}{/}g; + my $line = "$type $size $name\n"; + print $fh $line; + $list_txt .= $line; + $total_usage += $size; + } + close($pipe); + close($fh); + rename "$files.new", $files; + + open(my $usage, '>', "$files.usage.new"); + print $usage $total_usage; + close($usage); + rename "$files.usage.new", "$files.usage"; + + warn "# list_files $dir usage: $total_usage\n"; + + return $list_txt; +} + +sub refresh_file_list { + my ( $self, $user ) = @_; + $user = $self->user_info($user) unless ref $user eq 'HASH'; + my $full_path = "$user->{dir}/.meta/files"; + if ( -e $full_path ) { + warn "## refresh_file_list $full_path"; + unlink $full_path || warn "unlink $full_path: $!"; + } else { + warn "## refresh_file_list $full_path missing"; + } + + unlink "$full_path.usage" if -e "$full_path.usage"; } 1;