From f5ef8b2546333a4b7c7cf27ae62542346fbeeef5 Mon Sep 17 00:00:00 2001 From: Dobrica Pavlinusic Date: Thu, 27 Jan 2011 00:10:33 +0000 Subject: [PATCH] first steps in importing archive data to RDBMS --- bin/BackupPC_ASA_PostArchive_Update | 134 ++++++++++++++-------------- 1 file changed, 66 insertions(+), 68 deletions(-) diff --git a/bin/BackupPC_ASA_PostArchive_Update b/bin/BackupPC_ASA_PostArchive_Update index 1ab0965..97c1318 100755 --- a/bin/BackupPC_ASA_PostArchive_Update +++ b/bin/BackupPC_ASA_PostArchive_Update @@ -27,8 +27,16 @@ BackupPC_ASA_PostArchive_Update =cut +# FIXME +my $debug = $ENV{DEBUG} || 1; +my $check = $ENV{CHECK} || 1; + + +my $bpc = BackupPC::Lib->new || die "can't create BackupPC::Lib"; +my %Conf = $bpc->Conf(); warn "## ARGV=",dump @ARGV; + my $args; my $name; foreach ( @ARGV ) { @@ -46,13 +54,19 @@ foreach ( @ARGV ) { warn "args = ",dump($args); -# FIXME -my $debug = $ENV{DEBUG} || 1; -my $check = 0; +sub check_archive; +foreach ( 0 .. $#{ $args->{HostList} } ) { + + my $host = $args->{'HostList'}->[$_]; + my $num = $args->{'BackupList'}->[$_]; + + check_archive $host => $num; + +} + +exit; -my $bpc = BackupPC::Lib->new || die "can't create BackupPC::Lib"; -my %Conf = $bpc->Conf(); use BackupPC::Search; %BackupPC::Search::Conf = %Conf; @@ -66,6 +80,8 @@ my $start_t = time(); my $t_fmt = '%Y-%m-%d %H:%M:%S'; +warn "## Conf = ",dump( \%Conf ); + my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n"; my $user = $Conf{SearchUser} || ''; @@ -87,8 +103,6 @@ sub fmt_time { return $out; } -=for later - my $hsn_cache; sub get_backup_id($$$) { @@ -110,7 +124,7 @@ sub get_backup_id($$$) { $hsn_cache->{"$host $share $num"} = $id; - print STDERR "# $host $share $num == $id\n" if ($opt{d}); + print STDERR "# $host $share $num == $id\n" if $debug; return $id; } @@ -125,11 +139,31 @@ sub backup_inc_deleted($) { $sth_inc_deleted->execute($backup_id); } -sub tar_check($$$$) { - my ($host,$share,$num,$filename) = @_; +sub system_ok { + warn "## system_ok @_\n"; + system(@_) == 0 || die "system @_:$!"; +} + +sub check_archive { + my ($host,$num) = @_; my $t = time(); - print curr_time, " check $host:$share#$num -> $filename"; + print curr_time, " check $host $num"; + + my $md5_path = "$Conf{ArchiveDest}/$host.$num.md5"; + unlink $md5_path if -s $md5_path == 0; # fix empty + + if ( ! -e $md5_path ) { + system_ok "md5sum $Conf{ArchiveDest}/$host.$num.* > $md5_path"; + } else { + system_ok "md5sum -c $md5_path" if $check; + } + + my $md5sum; + foreach ( split(/\n/, read_file "$Conf{ArchiveDest}/$host.$num.md5" ) ) { + my ( $path, $md5 ) = split(/\s+/,$_); + $md5sum->{$path} = $md5; + } # depending on expected returned value this is used like: # my $uncompress_size = get_gzip_size('/full/path/to.gz'); @@ -137,7 +171,7 @@ sub tar_check($$$$) { sub get_gzip_size($) { my $filename = shift; die "file $filename problem: $!" unless (-r $filename); - open(my $gzip, $bin->{gzip}." -l $filename |") || die "can't gzip -l $filename: $!"; + open(my $gzip, $Conf{GzipPath}." -l $filename |") || die "can't gzip -l $filename: $!"; my $line = <$gzip>; chomp($line); $line = <$gzip> if ($line =~ /^\s+compressed/); @@ -193,31 +227,26 @@ sub tar_check($$$$) { $dbh->commit; } - my @tar_parts; - - if (-d "$tar_dir/$filename") { - print ", multi-part"; - opendir(my $dir, "$tar_dir/$filename") || die "can't readdir $tar_dir/$filename: $!"; - @tar_parts = map { my $p = $_; $p =~ s#^#${filename}/#; $p } grep { !/^\./ && !/md5/ && -f "$tar_dir/$filename/$_" } readdir($dir); - closedir($dir); - } else { - push @tar_parts, "${filename}.tar.gz"; - } - - print " [parts: ",join(", ", @tar_parts),"]" if ($opt{d}); + my @tar_parts = + sort map { s/^\Q$Conf{ArchiveDest}\E//; $_ } + glob "$Conf{ArchiveDest}/$host.$num.*" + ; + print " [parts: ",join(", ", @tar_parts),"]" if $debug; my $same = 1; my @tar_files; my $backup_part; - print " reading" if ($opt{d}); + print " reading" if $debug; + + my $part_nr = 1; foreach my $tarfilename (@tar_parts) { print "\n\t- $tarfilename"; - my $path = "$tar_dir/$tarfilename"; + my $path = "$Conf{ArchiveDest}/$tarfilename"; my $size = (stat( $path ))[7] || die "can't stat $path: $!"; @@ -228,6 +257,7 @@ sub tar_check($$$$) { print ", $size bytes"; +=for later open(my $fh, "gzip -cd $path |") or die "can't open $path: $!"; binmode($fh); @@ -263,39 +293,21 @@ sub tar_check($$$$) { return 1; } - my $tar_size = get_gzip_size( $path ); +=cut - # real tar size is bigger because of padding - if ($tar_size_inarc > $tar_size) { - print ", size of files in tar ($tar_size_inarc) bigger than whole tar ($tar_size)!\n"; - return 0; - } + # FIXME + my $tar_size = $path =~ m/.gz/ ? get_gzip_size( $path ) : -s $path; # - # check if md5 exists, and if not, create one + # finally, check if backup_parts table in database is valid # - my $md5_path = $path; - $md5_path =~ s/\.tar\.gz$/.md5/ || die "can't create md5 filename from $md5_path"; - if (! -e $md5_path || -z $md5_path) { - print ", creating md5"; - system( $bin->{md5sum} . " $path > $md5_path") == 0 or die "can't create md5 $path: $!"; - } else { - ## FIXME check if existing md5 is valid - } + my $md5 = $md5sum->{$path} || die "no md5sum for $path"; + my $items = 1; - my $md5 = read_file( $md5_path ) || die "can't read md5sum file $md5_path: $!"; - $md5 =~ s#\s.*$##; + check_part($host, undef, $num, $part_nr, $tar_size, $size, $md5, $items); - # extract part number from filename - my $part_nr = 1; - $part_nr = $1 if ($tarfilename =~ m#/(\d+)\.tar\.gz#); - - # - # finally, check if backup_parts table in database is valid - # - - check_part($host, $share, $num, $part_nr, $tar_size, $size, $md5, $items); + $part_nr++; } # short-cut and exit; @@ -309,9 +321,9 @@ sub tar_check($$$$) { FROM files JOIN shares on shares.id = shareid JOIN hosts on hosts.id = shares.hostid - WHERE hosts.name = ? and shares.name = ? and backupnum = ? + WHERE hosts.name = ? and backupnum = ? }); - $sth->execute($host, $share, $num); + $sth->execute($host, $num); my @db_files; while( my $row = $sth->fetchrow_hashref ) { @@ -344,7 +356,6 @@ sub tar_check($$$$) { return $same; } -=cut #----- main @@ -376,19 +387,6 @@ my $curr_backup = 1; while (my $row = $sth->fetchrow_hashref) { - my $found; - foreach ( 0 .. $#{ $args->{HostList} } ) { - if ( $args->{'HostList'}->[$_] eq $row->{host} - && $args->{'BackupList'}->[$_] eq $row->{num} ) { - $found = 1; - } - } - - if ( ! $found ) { - warn "skipped ",dump($row); - next; - } - $curr_backup++; my $tar_file = BackupPC::Search::getGzipName($row->{'host'}, $row->{'share'}, $row->{'num'}); -- 2.20.1