From: Dobrica Pavlinusic Date: Thu, 27 Jan 2011 01:21:58 +0000 (+0000) Subject: first version which stores backup parts X-Git-Url: http://git.rot13.org/?p=BackupPC.git;a=commitdiff_plain;h=dfc71d1e230596fcb539889fce2a475c45e0acd0;hp=6e9be0d94e5d99b8259d3801c638728eaccde5c4 first version which stores backup parts but doesn't work with consistency triggers enabled --- diff --git a/bin/BackupPC_ASA_PostArchive_Update b/bin/BackupPC_ASA_PostArchive_Update index 97c1318..aff3a0b 100755 --- a/bin/BackupPC_ASA_PostArchive_Update +++ b/bin/BackupPC_ASA_PostArchive_Update @@ -7,13 +7,12 @@ use DBI; use BackupPC::Lib; use BackupPC::View; use BackupPC::Attrib qw/:all/; -#use Data::Dumper; use Data::Dump qw(dump); use Time::HiRes qw/time/; use POSIX qw/strftime/; use Cwd qw/abs_path/; -#use Archive::Tar::Streamed; -#use Algorithm::Diff; +use Archive::Tar::Streamed; +use Algorithm::Diff; use Getopt::Std; use File::Slurp; @@ -54,19 +53,6 @@ foreach ( @ARGV ) { warn "args = ",dump($args); -sub check_archive; - -foreach ( 0 .. $#{ $args->{HostList} } ) { - - my $host = $args->{'HostList'}->[$_]; - my $num = $args->{'BackupList'}->[$_]; - - check_archive $host => $num; - -} - -exit; - use BackupPC::Search; %BackupPC::Search::Conf = %Conf; @@ -105,10 +91,10 @@ sub fmt_time { my $hsn_cache; -sub get_backup_id($$$) { - my ($host, $share, $num) = @_; +sub get_backup_id($$) { + my ($host, $num) = @_; - my $key = "$host $share $num"; + my $key = "$host $num"; return $hsn_cache->{$key} if ($hsn_cache->{$key}); my $sth = $dbh->prepare(qq{ @@ -117,14 +103,14 @@ sub get_backup_id($$$) { FROM backups INNER JOIN shares ON backups.shareID=shares.ID INNER JOIN hosts ON backups.hostID = hosts.ID - WHERE hosts.name = ? and shares.name = ? and backups.num = ? + WHERE hosts.name = ? and backups.num = ? }); - $sth->execute($host, $share, $num); + $sth->execute($host, $num); my ($id) = $sth->fetchrow_array; - $hsn_cache->{"$host $share $num"} = $id; + $hsn_cache->{"$host $num"} = $id; - print STDERR "# $host $share $num == $id\n" if $debug; + print STDERR "# $host $num == $id\n" if $debug; return $id; } @@ -148,20 +134,28 @@ sub check_archive { my ($host,$num) = @_; my $t = time(); + + my @tar_parts = + sort map { s/^\Q$Conf{ArchiveDest}\E\/*//; $_ } + glob "$Conf{ArchiveDest}/$host.$num.*" + ; + + return unless @tar_parts; + print curr_time, " check $host $num"; my $md5_path = "$Conf{ArchiveDest}/$host.$num.md5"; unlink $md5_path if -s $md5_path == 0; # fix empty if ( ! -e $md5_path ) { - system_ok "md5sum $Conf{ArchiveDest}/$host.$num.* > $md5_path"; + system_ok "cd $Conf{ArchiveDest} && /usr/bin/md5sum $host.$num.* > $md5_path"; } else { - system_ok "md5sum -c $md5_path" if $check; + system_ok "cd $Conf{ArchiveDest} && /usr/bin/md5sum -c $md5_path" if $check; } my $md5sum; foreach ( split(/\n/, read_file "$Conf{ArchiveDest}/$host.$num.md5" ) ) { - my ( $path, $md5 ) = split(/\s+/,$_); + my ( $md5, $path ) = split(/\s+/,$_); $md5sum->{$path} = $md5; } @@ -171,6 +165,11 @@ sub check_archive { sub get_gzip_size($) { my $filename = shift; die "file $filename problem: $!" unless (-r $filename); + + if ( $filename !~ m/\.gz$/ ) { + return -s $filename; + } + open(my $gzip, $Conf{GzipPath}." -l $filename |") || die "can't gzip -l $filename: $!"; my $line = <$gzip>; chomp($line); @@ -190,8 +189,8 @@ sub check_archive { } sub check_part { - my ($host, $share, $num, $part_nr, $tar_size, $size, $md5, $items) = @_; - my $backup_id = get_backup_id($host, $share, $num); + my ($host, $num, $part_nr, $tar_size, $size, $md5, $items) = @_; + my $backup_id = get_backup_id($host, $num); my $sth_md5 = $dbh->prepare(qq{ select id, tar_size, size, md5, items @@ -227,10 +226,6 @@ sub check_archive { $dbh->commit; } - my @tar_parts = - sort map { s/^\Q$Conf{ArchiveDest}\E//; $_ } - glob "$Conf{ArchiveDest}/$host.$num.*" - ; print " [parts: ",join(", ", @tar_parts),"]" if $debug; my $same = 1; @@ -242,11 +237,14 @@ sub check_archive { my $part_nr = 1; - foreach my $tarfilename (@tar_parts) { + foreach my $filename (@tar_parts) { + + next if $filename eq "$host.$num.md5"; - print "\n\t- $tarfilename"; + print "\n\t- $filename"; - my $path = "$Conf{ArchiveDest}/$tarfilename"; + my $path = "$Conf{ArchiveDest}/$filename"; + $path =~ s{//+}{/}g; my $size = (stat( $path ))[7] || die "can't stat $path: $!"; @@ -272,7 +270,7 @@ sub check_archive { $tar_size_inarc += $entry->size; if ($tar_size_inarc > $Conf{MaxArchiveFileSize}) { - print ", part $tarfilename is too big $tar_size_inarc > $Conf{MaxArchiveFileSize}\n"; + print ", part $filename is too big $tar_size_inarc > $Conf{MaxArchiveFileSize}\n"; return 0; } @@ -296,16 +294,16 @@ sub check_archive { =cut # FIXME - my $tar_size = $path =~ m/.gz/ ? get_gzip_size( $path ) : -s $path; + my $tar_size = get_gzip_size( $path ); # # finally, check if backup_parts table in database is valid # - my $md5 = $md5sum->{$path} || die "no md5sum for $path"; + my $md5 = $md5sum->{$filename} || die "no md5sum for $filename in ",dump($md5sum); my $items = 1; - check_part($host, undef, $num, $part_nr, $tar_size, $size, $md5, $items); + check_part($host, $num, $part_nr, $tar_size, $size, $md5, $items); $part_nr++; } @@ -359,6 +357,17 @@ sub check_archive { #----- main +foreach ( 0 .. $#{ $args->{HostList} } ) { + + my $host = $args->{'HostList'}->[$_]; + my $num = $args->{'BackupList'}->[$_]; + + check_archive $host => $num; + +} + +exit; + my $sth = $dbh->prepare( qq{ select diff --git a/bin/BackupPC_ASA_SearchUpdate b/bin/BackupPC_ASA_SearchUpdate index 20d5ccd..ede665f 100755 --- a/bin/BackupPC_ASA_SearchUpdate +++ b/bin/BackupPC_ASA_SearchUpdate @@ -359,6 +359,8 @@ if ($opt{c}) { $dbh->do( qq{ CREATE SEQUENCE $seq } ); } +=cut + print " creating triggers "; $dbh->do( <<__END_OF_TRIGGER__ ); @@ -420,6 +422,8 @@ create trigger do_backup_backup_parts_check __END_OF_TRIGGER__ +=cut + print "...\n"; $dbh->commit;