first steps in importing archive data to RDBMS

diff --git a/bin/BackupPC_ASA_PostArchive_Update b/bin/BackupPC_ASA_PostArchive_Update
index 1ab0965..97c1318 100755 (executable)
--- a/bin/BackupPC_ASA_PostArchive_Update
+++ b/bin/BackupPC_ASA_PostArchive_Update
@@ -27,8 +27,16 @@ BackupPC_ASA_PostArchive_Update
 
 =cut
 
+# FIXME
+my $debug = $ENV{DEBUG} || 1;
+my $check = $ENV{CHECK} || 1;
+
+
+my $bpc = BackupPC::Lib->new || die "can't create BackupPC::Lib";
+my %Conf = $bpc->Conf();
 warn "## ARGV=",dump @ARGV;
 
+
 my $args;
 my $name;
 foreach ( @ARGV ) {
@@ -46,13 +54,19 @@ foreach ( @ARGV ) {
 
 warn "args = ",dump($args);
 
-# FIXME
-my $debug = $ENV{DEBUG} || 1;
-my $check = 0;
+sub check_archive;
 
+foreach ( 0 .. $#{ $args->{HostList} } ) {
+
+       my $host = $args->{'HostList'}->[$_];
+       my $num  = $args->{'BackupList'}->[$_];
+
+       check_archive $host => $num;
+
+}
+
+exit;
 
-my $bpc = BackupPC::Lib->new || die "can't create BackupPC::Lib";
-my %Conf = $bpc->Conf();
 
 use BackupPC::Search;
 %BackupPC::Search::Conf = %Conf;
@@ -66,6 +80,8 @@ my $start_t = time();
 
 my $t_fmt = '%Y-%m-%d %H:%M:%S';
 
+warn "## Conf = ",dump( \%Conf );
+
 my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n";
 my $user = $Conf{SearchUser} || '';
 
@@ -87,8 +103,6 @@ sub fmt_time {
        return $out;
 }
 
-=for later
-
 my $hsn_cache;
 
 sub get_backup_id($$$) {
@@ -110,7 +124,7 @@ sub get_backup_id($$$) {
 
        $hsn_cache->{"$host $share $num"} = $id;
 
-       print STDERR "# $host $share $num == $id\n" if ($opt{d});
+       print STDERR "# $host $share $num == $id\n" if $debug;
 
        return $id;
 }
@@ -125,11 +139,31 @@ sub backup_inc_deleted($) {
        $sth_inc_deleted->execute($backup_id);
 }
 
-sub tar_check($$$$) {
-       my ($host,$share,$num,$filename) = @_;
+sub system_ok {
+       warn "## system_ok @_\n";
+       system(@_) == 0 || die "system @_:$!";
+}
+
+sub check_archive {
+       my ($host,$num) = @_;
 
        my $t = time();
-       print curr_time, " check $host:$share#$num -> $filename";
+       print curr_time, " check $host $num";
+
+       my $md5_path = "$Conf{ArchiveDest}/$host.$num.md5";
+       unlink $md5_path if -s $md5_path == 0; # fix empty
+
+       if ( ! -e $md5_path ) {
+               system_ok "md5sum $Conf{ArchiveDest}/$host.$num.* > $md5_path";
+       } else {
+               system_ok "md5sum -c $md5_path" if $check;
+       }
+
+       my $md5sum;
+       foreach ( split(/\n/, read_file "$Conf{ArchiveDest}/$host.$num.md5" ) ) {
+               my ( $path, $md5 ) = split(/\s+/,$_);
+               $md5sum->{$path} = $md5;
+       }
 
        # depending on expected returned value this is used like:
        # my $uncompress_size = get_gzip_size('/full/path/to.gz');
@@ -137,7 +171,7 @@ sub tar_check($$$$) {
        sub get_gzip_size($) {
                my $filename = shift;
                die "file $filename problem: $!" unless (-r $filename);
-               open(my $gzip, $bin->{gzip}." -l $filename |") || die "can't gzip -l $filename: $!";
+               open(my $gzip, $Conf{GzipPath}." -l $filename |") || die "can't gzip -l $filename: $!";
                my $line = <$gzip>;
                chomp($line);
                $line = <$gzip> if ($line =~ /^\s+compressed/);
@@ -193,31 +227,26 @@ sub tar_check($$$$) {
                $dbh->commit;
        }
 
-       my @tar_parts;
-
-       if (-d "$tar_dir/$filename") {
-               print ", multi-part";
-               opendir(my $dir, "$tar_dir/$filename") || die "can't readdir $tar_dir/$filename: $!";
-               @tar_parts = map { my $p = $_; $p =~ s#^#${filename}/#; $p } grep { !/^\./ && !/md5/ && -f "$tar_dir/$filename/$_" } readdir($dir);
-               closedir($dir);
-       } else {
-               push @tar_parts, "${filename}.tar.gz";
-       }
-
-       print " [parts: ",join(", ", @tar_parts),"]" if ($opt{d});
+       my @tar_parts =
+               sort map { s/^\Q$Conf{ArchiveDest}\E//; $_ }
+               glob "$Conf{ArchiveDest}/$host.$num.*"
+               ;
+       print " [parts: ",join(", ", @tar_parts),"]" if $debug;
 
        my $same = 1;
        my @tar_files;
 
        my $backup_part;
 
-       print " reading" if ($opt{d});
+       print " reading" if $debug;
+
+       my $part_nr = 1;
 
        foreach my $tarfilename (@tar_parts) {
 
                print "\n\t- $tarfilename";
 
-               my $path = "$tar_dir/$tarfilename";
+               my $path = "$Conf{ArchiveDest}/$tarfilename";
 
                my $size = (stat( $path ))[7] || die "can't stat $path: $!";
 
@@ -228,6 +257,7 @@ sub tar_check($$$$) {
 
                print ", $size bytes";
 
+=for later
 
                open(my $fh, "gzip -cd $path |") or die "can't open $path: $!";
                binmode($fh);
@@ -263,39 +293,21 @@ sub tar_check($$$$) {
                        return 1;
                }
 
-               my $tar_size = get_gzip_size( $path );
+=cut
 
-               # real tar size is bigger because of padding    
-               if ($tar_size_inarc > $tar_size) {
-                       print ", size of files in tar ($tar_size_inarc) bigger than whole tar ($tar_size)!\n";
-                       return 0;
-               }
+               # FIXME
+               my $tar_size = $path =~ m/.gz/ ? get_gzip_size( $path ) : -s $path;
 
                #
-               # check if md5 exists, and if not, create one
+               # finally, check if backup_parts table in database is valid
                #
 
-               my $md5_path = $path;
-               $md5_path =~ s/\.tar\.gz$/.md5/ || die "can't create md5 filename from $md5_path";
-               if (! -e $md5_path || -z $md5_path) {
-                       print ", creating md5";
-                       system( $bin->{md5sum} . " $path > $md5_path") == 0 or die "can't create md5 $path: $!";
-               } else {
-                       ## FIXME check if existing md5 is valid
-               }
+               my $md5 = $md5sum->{$path} || die "no md5sum for $path";
+               my $items = 1;
 
-               my $md5 = read_file( $md5_path ) || die "can't read md5sum file $md5_path: $!";
-               $md5 =~ s#\s.*$##;
+               check_part($host, undef, $num, $part_nr, $tar_size, $size, $md5, $items);
 
-               # extract part number from filename
-               my $part_nr = 1;
-               $part_nr = $1 if ($tarfilename =~ m#/(\d+)\.tar\.gz#);
-
-               #
-               # finally, check if backup_parts table in database is valid
-               #
-
-               check_part($host, $share, $num, $part_nr, $tar_size, $size, $md5, $items);
+               $part_nr++;
        }
 
        # short-cut and exit;
@@ -309,9 +321,9 @@ sub tar_check($$$$) {
                FROM files
                JOIN shares on shares.id = shareid
                JOIN hosts on hosts.id = shares.hostid
-               WHERE hosts.name = ? and shares.name = ? and backupnum = ?
+               WHERE hosts.name = ? and backupnum = ?
        });
-       $sth->execute($host, $share, $num);
+       $sth->execute($host, $num);
        my @db_files;
        while( my $row = $sth->fetchrow_hashref ) {
 
@@ -344,7 +356,6 @@ sub tar_check($$$$) {
        return $same;
 }
 
-=cut
 
 #----- main
 
@@ -376,19 +387,6 @@ my $curr_backup = 1;
 
 while (my $row = $sth->fetchrow_hashref) {
 
-       my $found;
-       foreach ( 0 .. $#{ $args->{HostList} } ) {
-               if ( $args->{'HostList'}->[$_] eq $row->{host}
-               &&   $args->{'BackupList'}->[$_] eq $row->{num} ) {
-                       $found = 1;
-               }
-       }
-
-       if ( ! $found ) {
-               warn "skipped ",dump($row);
-               next;
-       }
-
        $curr_backup++;
 
        my $tar_file = BackupPC::Search::getGzipName($row->{'host'}, $row->{'share'}, $row->{'num'});