X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;f=bin%2FBackupPC_incPartsUpdate;h=72a5ba0cfc723b29d949cb19cf5bcfb4c3dd8283;hb=d3f1209a76a69f642ffbfe2fb796ebdf650753e7;hp=b7746c27b4cef706bd7d66a706a6927573e9a305;hpb=c548262329b0b6e1e1a3daeb0f071ba2c6e4c62e;p=BackupPC.git diff --git a/bin/BackupPC_incPartsUpdate b/bin/BackupPC_incPartsUpdate index b7746c2..72a5ba0 100755 --- a/bin/BackupPC_incPartsUpdate +++ b/bin/BackupPC_incPartsUpdate @@ -16,6 +16,58 @@ use Archive::Tar::Streamed; use Algorithm::Diff; use Getopt::Std; use File::Slurp; +use File::Pid; + +=head1 NAME + +BackupPC_incPartsUpdate + +=head1 DESCRIPTION + +Create C<.tar.gz> increments on disk calling C. + +Following options are supported (but all are optional): + +=over 4 + +=item -h hostname + +Update parts for just single C + +=item -c + +Force check for tar archives which exist on disk + +=item -d + +Turn debugging output + +=back + +=cut + +my %opt; +getopts("cdh:", \%opt ); + +my $debug = $opt{d}; +my $check = $opt{c} && print STDERR "NOTICE: tar archive check forced\n"; + +my $pid_path = abs_path($0); +$pid_path =~ s/\W+/_/g; + +my $pidfile = new File::Pid({ + file => "/tmp/$pid_path", +}); + +if (my $pid = $pidfile->running ) { + die "$0 already running: $pid\n"; +} elsif ($pidfile->pid ne $$) { + $pidfile->remove; + $pidfile = new File::Pid; +} + +print STDERR "$0 using pid ",$pidfile->pid," file ",$pidfile->file,"\n"; +$pidfile->write; my $bpc = BackupPC::Lib->new || die "can't create BackupPC::Lib"; my %Conf = $bpc->Conf(); @@ -34,12 +86,6 @@ foreach my $c (qw/gzip md5sum/) { $bin->{$c} = which($c) || die "$0 needs $c, install it\n"; } -my %opt; -getopts("cd", \%opt ); - -my $debug = $opt{d}; -my $check = $opt{c} && print STDERR "NOTICE: tar archive check forced\n"; - $|=1; my $start_t = time(); @@ -84,7 +130,7 @@ sub get_backup_id($$$) { FROM backups INNER JOIN shares ON backups.shareID=shares.ID INNER JOIN hosts ON backups.hostID = hosts.ID - where hosts.name = ? and shares.name = ? and backups.num = ? + WHERE hosts.name = ? and shares.name = ? and backups.num = ? }); $sth->execute($host, $share, $num); my ($id) = $sth->fetchrow_array; @@ -96,6 +142,15 @@ sub get_backup_id($$$) { return $id; } +sub backup_inc_deleted($) { + my $backup_id = shift; + my $sth_inc_deleted = $dbh->prepare(qq{ + update backups set + inc_deleted = true + where id = ? + }); + $sth_inc_deleted->execute($backup_id); +} sub tar_check($$$$) { my ($host,$share,$num,$filename) = @_; @@ -103,6 +158,30 @@ sub tar_check($$$$) { my $t = time(); print curr_time, " check $host:$share#$num -> $filename"; + # depending on expected returned value this is used like: + # my $uncompress_size = get_gzip_size('/full/path/to.gz'); + # my ($compress_size, $uncompress_size) = get_gzip_size('/path.gz'); + sub get_gzip_size($) { + my $filename = shift; + die "file $filename problem: $!" unless (-r $filename); + open(my $gzip, $bin->{gzip}." -l $filename |") || die "can't gzip -l $filename: $!"; + my $line = <$gzip>; + chomp($line); + $line = <$gzip> if ($line =~ /^\s+compressed/); + + my ($comp, $uncomp) = (0,0); + + if ($line =~ m/^\s+(\d+)\s+(\d+)\s+\d+\.\d+/) { + if (wantarray) { + return [ $1, $2 ]; + } else { + return $2; + } + } else { + die "can't find size in line: $line"; + } + } + sub check_part { my ($host, $share, $num, $part_nr, $tar_size, $size, $md5, $items) = @_; my $backup_id = get_backup_id($host, $share, $num); @@ -165,7 +244,9 @@ sub tar_check($$$$) { print "\n\t- $tarfilename"; - my $size = (stat( "$tar_dir/$tarfilename" ))[7] || die "can't stat $tar_dir/$tarfilename"; + my $path = "$tar_dir/$tarfilename"; + + my $size = (stat( $path ))[7] || die "can't stat $path: $!"; if ($size > $Conf{MaxArchiveSize}) { print ", part bigger than media $size > $Conf{MaxArchiveSize}\n"; @@ -174,49 +255,48 @@ sub tar_check($$$$) { print ", $size bytes"; - my $path = "$tar_dir/$tarfilename"; - open(my $fh, "gzip -cd $tar_dir/$tarfilename |") or die "can't open $tar_dir/$tarfilename: $!"; + open(my $fh, "gzip -cd $path |") or die "can't open $path: $!"; binmode($fh); my $tar = Archive::Tar::Streamed->new($fh); - my $tar_size = 0; + my $tar_size_inarc = 0; my $items = 0; while(my $entry = $tar->next) { push @tar_files, $entry->name; $items++; - $tar_size += $entry->size; + $tar_size_inarc += $entry->size; - if ($tar_size > $Conf{MaxArchiveFileSize}) { - print ", part $tarfilename is too big $tar_size > $Conf{MaxArchiveFileSize}\n"; + if ($tar_size_inarc > $Conf{MaxArchiveFileSize}) { + print ", part $tarfilename is too big $tar_size_inarc > $Conf{MaxArchiveFileSize}\n"; return 0; } } + close($fh); + print ", $items items"; - if ($tar_size == 0 && $items == 0) { + if ($tar_size_inarc == 0 && $items == 0) { print ", EMPTY tar\n"; my $backup_id = get_backup_id($host, $share, $num); - - my $sth_inc_deleted = $dbh->prepare(qq{ - update backups set - inc_deleted = true - where id = ? - }); - $sth_inc_deleted->execute($backup_id); + backup_inc_deleted( $backup_id ); $dbh->commit; return 1; } - # fix tar_size for tars without any files - $tar_size ||= 512 * $items; + my $tar_size = get_gzip_size( $path ); + # real tar size is bigger because of padding + if ($tar_size_inarc > $tar_size) { + print ", size of files in tar ($tar_size_inarc) bigger than whole tar ($tar_size)!\n"; + return 0; + } # # check if md5 exists, and if not, create one @@ -301,13 +381,17 @@ select hosts.name as host, shares.name as share, backups.num as num, + backups.date, inc_size, - parts + parts, + count(backup_parts.backup_id) as backup_parts from backups join shares on backups.hostid = shares.hostid and shares.id = backups.shareid join hosts on shares.hostid = hosts.id -where not inc_deleted + full outer join backup_parts on backups.id = backup_parts.backup_id +where not inc_deleted and backups.size > 0 +group by backups.id, hosts.name, shares.name, backups.num, backups.date, inc_size, parts, backup_parts.backup_id order by backups.date } ); @@ -316,8 +400,17 @@ $sth->execute(); my $num_backups = $sth->rows; my $curr_backup = 1; +if ($opt{h}) { + warn "making increments just for host $opt{h}\n"; +} + while (my $row = $sth->fetchrow_hashref) { + if ($opt{h} && $row->{host} ne $opt{h}) { + warn "skipped $row->{host}\n" if ($debug); + next; + } + $curr_backup++; my $tar_file = BackupPC::SearchLib::getGzipName($row->{'host'}, $row->{'share'}, $row->{'num'}); @@ -327,7 +420,7 @@ while (my $row = $sth->fetchrow_hashref) { print "# size: $size backup.size: ", $row->{inc_size},"\n" if ($opt{d}); - if ( $row->{'inc_size'} != -1 && $size != -1 && $row->{'inc_size'} >= $size) { + if ( $row->{'inc_size'} != -1 && $size != -1 && $row->{'inc_size'} >= $size && $row->{parts} == $row->{backup_parts}) { if ($check) { tar_check($row->{'host'}, $row->{'share'}, $row->{'num'}, $tar_file) && next; } else { @@ -335,16 +428,18 @@ while (my $row = $sth->fetchrow_hashref) { } } - print curr_time, " creating $curr_backup/$num_backups ", $row->{'host'}, ":", $row->{'share'}, " #", $row->{'num'}, " -> $tar_file"; + print curr_time, " creating $curr_backup/$num_backups ", $row->{host}, ":", $row->{share}, " #", $row->{num}, + " ", strftime('%Y-%m-%d', localtime($row->{date})), " -> $tar_file"; my $t = time(); # re-create archive? - my $cmd = qq{ $tarIncCreate -h "$row->{'host'}" -s "$row->{'share'}" -n $row->{'num'} -f }; + my $cmd = qq[ $tarIncCreate -h "$row->{host}" -s "$row->{share}" -n $row->{num} -f ]; print STDERR "## $cmd\n" if ($debug); if (system($cmd) != 0) { - print STDERR " FAILED"; + print STDERR " FAILED, marking this backup deleted"; + backup_inc_deleted( $row->{backup_id} ); } print ", dur: ",fmt_time(time() - $t), "\n";