X-Git-Url: http://git.rot13.org/?p=BackupPC.git;a=blobdiff_plain;f=bin%2FBackupPC_ASA_PostArchive_Update;h=93478c985afbe71e8a82da6b982f75c75567e35b;hp=1ab0965274988d8535bd25972937a034d1673909;hb=06017d5181d9c62f8cbd1c6d58d831f54d7d4ad1;hpb=7a26d7def76984372c2bb54a7dbb6975647c17e4 diff --git a/bin/BackupPC_ASA_PostArchive_Update b/bin/BackupPC_ASA_PostArchive_Update index 1ab0965..93478c9 100755 --- a/bin/BackupPC_ASA_PostArchive_Update +++ b/bin/BackupPC_ASA_PostArchive_Update @@ -7,15 +7,14 @@ use DBI; use BackupPC::Lib; use BackupPC::View; use BackupPC::Attrib qw/:all/; -#use Data::Dumper; use Data::Dump qw(dump); use Time::HiRes qw/time/; use POSIX qw/strftime/; use Cwd qw/abs_path/; -#use Archive::Tar::Streamed; -#use Algorithm::Diff; -use Getopt::Std; +use Archive::Tar::Streamed; +use Algorithm::Diff; use File::Slurp; +use Getopt::Long::Descriptive; =head1 NAME @@ -23,42 +22,28 @@ BackupPC_ASA_PostArchive_Update =head1 DESCRIPTION - # /etc/BackupPC/pc/dvd_tar.pl + # /etc/BackupPC/pc/_search_archive.pl =cut -warn "## ARGV=",dump @ARGV; - -my $args; -my $name; -foreach ( @ARGV ) { - my $v = $_; - if ( m/(\w+)=(.+)/ ) { - $name = $1; - $v = $2; - } - if ( $name =~ m/List/ ) { - push @{ $args->{$name} }, $v; - } else { - $args->{$name} = $v; - } -} - -warn "args = ",dump($args); - -# FIXME -my $debug = $ENV{DEBUG} || 1; -my $check = 0; - my $bpc = BackupPC::Lib->new || die "can't create BackupPC::Lib"; +$bpc->ConfigRead('_search_archive'); # read our configuration my %Conf = $bpc->Conf(); use BackupPC::Search; %BackupPC::Search::Conf = %Conf; -my $path = abs_path($0); -$path =~ s{/[^/]+$}{/}; # FIXME remove? +my ($opt,$usage) = describe_options( +"%c %o", +[ 'host|h=s@', "import just host(s)" ], +[ 'num|n=s@', "import just backup number(s)" ], +[ 'check|c', "check archives on disk and sync", { default => 1 } ], +[ 'debug|d', "debug", { default => 1 } ], +[ 'help', "show help" ], +); + +print($usage->text), exit if $opt->help; $|=1; @@ -66,8 +51,7 @@ my $start_t = time(); my $t_fmt = '%Y-%m-%d %H:%M:%S'; -my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n"; -my $user = $Conf{SearchUser} || ''; +#warn "## Conf = ",dump( \%Conf ); my $dbh = DBI->connect($Conf{SearchDSN}, $Conf{SearchUser}, "", { RaiseError => 1, AutoCommit => 0 }); @@ -87,14 +71,12 @@ sub fmt_time { return $out; } -=for later - my $hsn_cache; -sub get_backup_id($$$) { - my ($host, $share, $num) = @_; +sub get_backup_id($$) { + my ($host, $num) = @_; - my $key = "$host $share $num"; + my $key = "$host $num"; return $hsn_cache->{$key} if ($hsn_cache->{$key}); my $sth = $dbh->prepare(qq{ @@ -103,14 +85,14 @@ sub get_backup_id($$$) { FROM backups INNER JOIN shares ON backups.shareID=shares.ID INNER JOIN hosts ON backups.hostID = hosts.ID - WHERE hosts.name = ? and shares.name = ? and backups.num = ? + WHERE hosts.name = ? and backups.num = ? }); - $sth->execute($host, $share, $num); + $sth->execute($host, $num); my ($id) = $sth->fetchrow_array; - $hsn_cache->{"$host $share $num"} = $id; + $hsn_cache->{"$host $num"} = $id; - print STDERR "# $host $share $num == $id\n" if ($opt{d}); + print STDERR "# $host $num == $id\n" if $opt->debug; return $id; } @@ -125,11 +107,62 @@ sub backup_inc_deleted($) { $sth_inc_deleted->execute($backup_id); } -sub tar_check($$$$) { - my ($host,$share,$num,$filename) = @_; +sub system_ok { + warn "## system_ok @_\n"; + system(@_) == 0 || die "system @_:$!"; +} + +my $sth_inc_size = $dbh->prepare(qq{ + update backups set + inc_size = ?, + parts = ?, + inc_deleted = false + where id = ? +}); + +sub read_only { + my $full = shift; + my $perm = (stat $full)[2] & 0444; + warn sprintf("chmod %03o %s\n",$perm,$full); + chmod $perm, $full || die $!; +} + +sub check_archive { + my ($host,$num) = @_; + warn "# check_archive $host $num"; my $t = time(); - print curr_time, " check $host:$share#$num -> $filename"; + + my $glob = "$Conf{ArchiveDest}/$host.$num.*"; + + my @tar_parts = sort map { s/^\Q$Conf{ArchiveDest}\E\/*//; $_ } glob $glob ; + + if ( ! @tar_parts ) { + warn "ERROR: no files for $glob"; + return; + } + + print curr_time, " check $host $num"; + + my $md5_path = "$Conf{ArchiveDest}/$host.$num.md5"; + unlink $md5_path if -s $md5_path == 0; # fix empty + + my $read_protect = 0; + + if ( ! -e $md5_path ) { + system_ok "cd $Conf{ArchiveDest} && /usr/bin/md5sum $host.$num.* > $md5_path"; + read_only $md5_path; + $read_protect = 1; + } else { + system_ok "cd $Conf{ArchiveDest} && /usr/bin/md5sum -c $md5_path" if $opt->check; + } + + my $md5sum; + foreach ( split(/\n/, read_file "$Conf{ArchiveDest}/$host.$num.md5" ) ) { + my ( $md5, $path ) = split(/\s+/,$_); + $md5sum->{$path} = $md5; + read_only "$Conf{ArchiveDest}/$path" if $read_protect; + } # depending on expected returned value this is used like: # my $uncompress_size = get_gzip_size('/full/path/to.gz'); @@ -137,35 +170,41 @@ sub tar_check($$$$) { sub get_gzip_size($) { my $filename = shift; die "file $filename problem: $!" unless (-r $filename); - open(my $gzip, $bin->{gzip}." -l $filename |") || die "can't gzip -l $filename: $!"; + + if ( $filename !~ m/\.gz$/ ) { + return -s $filename; + } + + open(my $gzip, $Conf{GzipPath}." -l $filename |") || die "can't gzip -l $filename: $!"; + local $/ = undef; my $line = <$gzip>; - chomp($line); - $line = <$gzip> if ($line =~ /^\s+compressed/); + close($gzip); my ($comp, $uncomp) = (0,0); - if ($line =~ m/^\s+(\d+)\s+(\d+)\s+\d+\.\d+/) { + if ($line =~ m/\s+(\d+)\s+(\d+)\s+\d+\.\d+/s) { if (wantarray) { return [ $1, $2 ]; } else { return $2; } } else { - die "can't find size in line: $line"; + warn "ERROR can't parse: $line"; + return -s $filename; } } sub check_part { - my ($host, $share, $num, $part_nr, $tar_size, $size, $md5, $items) = @_; - my $backup_id = get_backup_id($host, $share, $num); + my ($host, $num, $part_nr, $tar_size, $size, $md5, $items, $filename) = @_; + my $backup_id = get_backup_id($host, $num); my $sth_md5 = $dbh->prepare(qq{ select - id, tar_size, size, md5, items + id, tar_size, size, md5, items, filename from backup_parts - where backup_id = ? and part_nr = ? + where backup_id = ? and part_nr = ? and filename = ? }); - $sth_md5->execute($backup_id, $part_nr); + $sth_md5->execute($backup_id, $part_nr, $filename); if (my $row = $sth_md5->fetchrow_hashref) { return if ( @@ -185,49 +224,45 @@ sub tar_check($$$$) { tar_size, size, md5, - items - ) values (?,?,?,?,?,?) + items, + filename + ) values (?,?,?,?,?,?,?) }); - $sth_insert->execute($backup_id, $part_nr, $tar_size, $size, $md5, $items); + $sth_insert->execute($backup_id, $part_nr, $tar_size, $size, $md5, $items, $filename); $dbh->commit; } - my @tar_parts; - - if (-d "$tar_dir/$filename") { - print ", multi-part"; - opendir(my $dir, "$tar_dir/$filename") || die "can't readdir $tar_dir/$filename: $!"; - @tar_parts = map { my $p = $_; $p =~ s#^#${filename}/#; $p } grep { !/^\./ && !/md5/ && -f "$tar_dir/$filename/$_" } readdir($dir); - closedir($dir); - } else { - push @tar_parts, "${filename}.tar.gz"; - } - - print " [parts: ",join(", ", @tar_parts),"]" if ($opt{d}); + print " [parts: ",join(", ", @tar_parts),"]" if $opt->debug; - my $same = 1; my @tar_files; my $backup_part; - print " reading" if ($opt{d}); + print " reading" if $opt->debug; + + my $part_nr = 0; + my $inc_size = 0; - foreach my $tarfilename (@tar_parts) { + foreach my $filename (@tar_parts) { - print "\n\t- $tarfilename"; + next if $filename eq "$host.$num.md5"; - my $path = "$tar_dir/$tarfilename"; + print "\n\t- $filename"; + + my $path = "$Conf{ArchiveDest}/$filename"; + $path =~ s{//+}{/}g; my $size = (stat( $path ))[7] || die "can't stat $path: $!"; - if ($size > $Conf{MaxArchiveSize}) { - print ", part bigger than media $size > $Conf{MaxArchiveSize}\n"; + if ($size > $Conf{ArchiveMediaSize}) { + print ", part bigger than media $size > $Conf{ArchiveMediaSize}\n"; return 0; } print ", $size bytes"; +=for later open(my $fh, "gzip -cd $path |") or die "can't open $path: $!"; binmode($fh); @@ -241,8 +276,8 @@ sub tar_check($$$$) { $items++; $tar_size_inarc += $entry->size; - if ($tar_size_inarc > $Conf{MaxArchiveFileSize}) { - print ", part $tarfilename is too big $tar_size_inarc > $Conf{MaxArchiveFileSize}\n"; + if ($tar_size_inarc > $Conf{ArchiveMediaSize}) { + print ", part $filename is too big $tar_size_inarc > $Conf{ArchiveMediaSize}\n"; return 0; } @@ -263,43 +298,31 @@ sub tar_check($$$$) { return 1; } - my $tar_size = get_gzip_size( $path ); +=cut - # real tar size is bigger because of padding - if ($tar_size_inarc > $tar_size) { - print ", size of files in tar ($tar_size_inarc) bigger than whole tar ($tar_size)!\n"; - return 0; - } + # FIXME + my $tar_size = get_gzip_size( $path ); # - # check if md5 exists, and if not, create one + # finally, check if backup_parts table in database is valid # - my $md5_path = $path; - $md5_path =~ s/\.tar\.gz$/.md5/ || die "can't create md5 filename from $md5_path"; - if (! -e $md5_path || -z $md5_path) { - print ", creating md5"; - system( $bin->{md5sum} . " $path > $md5_path") == 0 or die "can't create md5 $path: $!"; - } else { - ## FIXME check if existing md5 is valid - } - - my $md5 = read_file( $md5_path ) || die "can't read md5sum file $md5_path: $!"; - $md5 =~ s#\s.*$##; + my $md5 = $md5sum->{$filename} || die "no md5sum for $filename in ",dump($md5sum); + my $items = 1; + $part_nr++; - # extract part number from filename - my $part_nr = 1; - $part_nr = $1 if ($tarfilename =~ m#/(\d+)\.tar\.gz#); + check_part($host, $num, $part_nr, $tar_size, $size, $md5, $items, $filename); - # - # finally, check if backup_parts table in database is valid - # - - check_part($host, $share, $num, $part_nr, $tar_size, $size, $md5, $items); + # round increment size to 2k block size + $inc_size += int((($size + 2048) / 2048 ) * 2048); } - # short-cut and exit; - return $same unless($same); + $sth_inc_size->execute( + $inc_size, + $part_nr, + get_backup_id($host, $num), + ); + $dbh->commit; @tar_files = sort @tar_files; print "\n\t",($#tar_files + 1), " tar files"; @@ -309,9 +332,9 @@ sub tar_check($$$$) { FROM files JOIN shares on shares.id = shareid JOIN hosts on hosts.id = shares.hostid - WHERE hosts.name = ? and shares.name = ? and backupnum = ? + WHERE hosts.name = ? and backupnum = ? }); - $sth->execute($host, $share, $num); + $sth->execute($host, $num); my @db_files; while( my $row = $sth->fetchrow_hashref ) { @@ -325,6 +348,8 @@ sub tar_check($$$$) { @db_files = sort @db_files; + my $same = 1; + if ($#tar_files != $#db_files) { $same = 0; print " NUMBER"; @@ -344,10 +369,20 @@ sub tar_check($$$$) { return $same; } -=cut #----- main +foreach ( 0 .. $#{ $opt->host } ) { + + my $host = $opt->host->[$_]; + my $num = $opt->num->[$_]; + + check_archive $host => $num; + +} + +exit; # FIXME + my $sth = $dbh->prepare( qq{ select @@ -376,19 +411,6 @@ my $curr_backup = 1; while (my $row = $sth->fetchrow_hashref) { - my $found; - foreach ( 0 .. $#{ $args->{HostList} } ) { - if ( $args->{'HostList'}->[$_] eq $row->{host} - && $args->{'BackupList'}->[$_] eq $row->{num} ) { - $found = 1; - } - } - - if ( ! $found ) { - warn "skipped ",dump($row); - next; - } - $curr_backup++; my $tar_file = BackupPC::Search::getGzipName($row->{'host'}, $row->{'share'}, $row->{'num'}); @@ -396,10 +418,10 @@ while (my $row = $sth->fetchrow_hashref) { # this will return -1 if file doesn't exist my $size = BackupPC::Search::get_tgz_size_by_name($tar_file); - print "# host: ".$row->{host}.", share: ".$row->{'share'}.", backup_num:".$row->{num}." size: $size backup.size: ", $row->{inc_size},"\n" if $debug; + print "# host: ".$row->{host}.", share: ".$row->{'share'}.", backup_num:".$row->{num}." size: $size backup.size: ", $row->{inc_size},"\n" if $opt->debug; if ( $row->{'inc_size'} != -1 && $size != -1 && $row->{'inc_size'} >= $size && $row->{parts} == $row->{backup_parts}) { - if ($check) { + if ($opt->check) { tar_check($row->{'host'}, $row->{'share'}, $row->{'num'}, $tar_file) && next; } else { next; @@ -414,7 +436,7 @@ while (my $row = $sth->fetchrow_hashref) { =for later # re-create archive? my $cmd = qq[ $tarIncCreate -h "$row->{host}" -s "$row->{share}" -n $row->{num} -f ]; - print STDERR "## $cmd\n" if ($debug); + print STDERR "## $cmd\n" if ($opt->debug); if (system($cmd) != 0) { print STDERR " FAILED, marking this backup deleted";