projects
/
BackupPC.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
if I can't find gzip size return original one
[BackupPC.git]
/
bin
/
BackupPC_ASA_PostArchive_Update
diff --git
a/bin/BackupPC_ASA_PostArchive_Update
b/bin/BackupPC_ASA_PostArchive_Update
index
97c1318
..
3a79676
100755
(executable)
--- a/
bin/BackupPC_ASA_PostArchive_Update
+++ b/
bin/BackupPC_ASA_PostArchive_Update
@@
-7,13
+7,12
@@
use DBI;
use BackupPC::Lib;
use BackupPC::View;
use BackupPC::Attrib qw/:all/;
use BackupPC::Lib;
use BackupPC::View;
use BackupPC::Attrib qw/:all/;
-#use Data::Dumper;
use Data::Dump qw(dump);
use Time::HiRes qw/time/;
use POSIX qw/strftime/;
use Cwd qw/abs_path/;
use Data::Dump qw(dump);
use Time::HiRes qw/time/;
use POSIX qw/strftime/;
use Cwd qw/abs_path/;
-
#
use Archive::Tar::Streamed;
-
#
use Algorithm::Diff;
+use Archive::Tar::Streamed;
+use Algorithm::Diff;
use Getopt::Std;
use File::Slurp;
use Getopt::Std;
use File::Slurp;
@@
-54,19
+53,6
@@
foreach ( @ARGV ) {
warn "args = ",dump($args);
warn "args = ",dump($args);
-sub check_archive;
-
-foreach ( 0 .. $#{ $args->{HostList} } ) {
-
- my $host = $args->{'HostList'}->[$_];
- my $num = $args->{'BackupList'}->[$_];
-
- check_archive $host => $num;
-
-}
-
-exit;
-
use BackupPC::Search;
%BackupPC::Search::Conf = %Conf;
use BackupPC::Search;
%BackupPC::Search::Conf = %Conf;
@@
-80,10
+66,7
@@
my $start_t = time();
my $t_fmt = '%Y-%m-%d %H:%M:%S';
my $t_fmt = '%Y-%m-%d %H:%M:%S';
-warn "## Conf = ",dump( \%Conf );
-
-my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n";
-my $user = $Conf{SearchUser} || '';
+#warn "## Conf = ",dump( \%Conf );
my $dbh = DBI->connect($Conf{SearchDSN}, $Conf{SearchUser}, "", { RaiseError => 1, AutoCommit => 0 });
my $dbh = DBI->connect($Conf{SearchDSN}, $Conf{SearchUser}, "", { RaiseError => 1, AutoCommit => 0 });
@@
-105,10
+88,10
@@
sub fmt_time {
my $hsn_cache;
my $hsn_cache;
-sub get_backup_id($$
$
) {
- my ($host, $
share, $
num) = @_;
+sub get_backup_id($$) {
+ my ($host, $num) = @_;
- my $key = "$host $
share $
num";
+ my $key = "$host $num";
return $hsn_cache->{$key} if ($hsn_cache->{$key});
my $sth = $dbh->prepare(qq{
return $hsn_cache->{$key} if ($hsn_cache->{$key});
my $sth = $dbh->prepare(qq{
@@
-117,14
+100,14
@@
sub get_backup_id($$$) {
FROM backups
INNER JOIN shares ON backups.shareID=shares.ID
INNER JOIN hosts ON backups.hostID = hosts.ID
FROM backups
INNER JOIN shares ON backups.shareID=shares.ID
INNER JOIN hosts ON backups.hostID = hosts.ID
- WHERE hosts.name = ? and
shares.name = ? and
backups.num = ?
+ WHERE hosts.name = ? and backups.num = ?
});
});
- $sth->execute($host, $
share, $
num);
+ $sth->execute($host, $num);
my ($id) = $sth->fetchrow_array;
my ($id) = $sth->fetchrow_array;
- $hsn_cache->{"$host $
share $
num"} = $id;
+ $hsn_cache->{"$host $num"} = $id;
- print STDERR "# $host $
share $
num == $id\n" if $debug;
+ print STDERR "# $host $num == $id\n" if $debug;
return $id;
}
return $id;
}
@@
-144,24
+127,40
@@
sub system_ok {
system(@_) == 0 || die "system @_:$!";
}
system(@_) == 0 || die "system @_:$!";
}
+my $sth_inc_size = $dbh->prepare(qq{
+ update backups set
+ inc_size = ?,
+ parts = ?,
+ inc_deleted = false
+ where id = ?
+});
+
sub check_archive {
my ($host,$num) = @_;
my $t = time();
sub check_archive {
my ($host,$num) = @_;
my $t = time();
+
+ my @tar_parts =
+ sort map { s/^\Q$Conf{ArchiveDest}\E\/*//; $_ }
+ glob "$Conf{ArchiveDest}/$host.$num.*"
+ ;
+
+ return unless @tar_parts;
+
print curr_time, " check $host $num";
my $md5_path = "$Conf{ArchiveDest}/$host.$num.md5";
unlink $md5_path if -s $md5_path == 0; # fix empty
if ( ! -e $md5_path ) {
print curr_time, " check $host $num";
my $md5_path = "$Conf{ArchiveDest}/$host.$num.md5";
unlink $md5_path if -s $md5_path == 0; # fix empty
if ( ! -e $md5_path ) {
- system_ok "
md5sum $Conf{ArchiveDest}/
$host.$num.* > $md5_path";
+ system_ok "
cd $Conf{ArchiveDest} && /usr/bin/md5sum
$host.$num.* > $md5_path";
} else {
} else {
- system_ok "md5sum -c $md5_path" if $check;
+ system_ok "
cd $Conf{ArchiveDest} && /usr/bin/
md5sum -c $md5_path" if $check;
}
my $md5sum;
foreach ( split(/\n/, read_file "$Conf{ArchiveDest}/$host.$num.md5" ) ) {
}
my $md5sum;
foreach ( split(/\n/, read_file "$Conf{ArchiveDest}/$host.$num.md5" ) ) {
- my ( $
path, $md5
) = split(/\s+/,$_);
+ my ( $
md5, $path
) = split(/\s+/,$_);
$md5sum->{$path} = $md5;
}
$md5sum->{$path} = $md5;
}
@@
-171,6
+170,11
@@
sub check_archive {
sub get_gzip_size($) {
my $filename = shift;
die "file $filename problem: $!" unless (-r $filename);
sub get_gzip_size($) {
my $filename = shift;
die "file $filename problem: $!" unless (-r $filename);
+
+ if ( $filename !~ m/\.gz$/ ) {
+ return -s $filename;
+ }
+
open(my $gzip, $Conf{GzipPath}." -l $filename |") || die "can't gzip -l $filename: $!";
my $line = <$gzip>;
chomp($line);
open(my $gzip, $Conf{GzipPath}." -l $filename |") || die "can't gzip -l $filename: $!";
my $line = <$gzip>;
chomp($line);
@@
-185,13
+189,14
@@
sub check_archive {
return $2;
}
} else {
return $2;
}
} else {
- die "can't find size in line: $line";
+ warn "ERROR can't parse: $line";
+ return -s $filename;
}
}
sub check_part {
}
}
sub check_part {
- my ($host, $
share, $
num, $part_nr, $tar_size, $size, $md5, $items) = @_;
- my $backup_id = get_backup_id($host, $
share, $
num);
+ my ($host, $num, $part_nr, $tar_size, $size, $md5, $items) = @_;
+ my $backup_id = get_backup_id($host, $num);
my $sth_md5 = $dbh->prepare(qq{
select
id, tar_size, size, md5, items
my $sth_md5 = $dbh->prepare(qq{
select
id, tar_size, size, md5, items
@@
-227,26
+232,25
@@
sub check_archive {
$dbh->commit;
}
$dbh->commit;
}
- my @tar_parts =
- sort map { s/^\Q$Conf{ArchiveDest}\E//; $_ }
- glob "$Conf{ArchiveDest}/$host.$num.*"
- ;
print " [parts: ",join(", ", @tar_parts),"]" if $debug;
print " [parts: ",join(", ", @tar_parts),"]" if $debug;
- my $same = 1;
my @tar_files;
my $backup_part;
print " reading" if $debug;
my @tar_files;
my $backup_part;
print " reading" if $debug;
- my $part_nr = 1;
+ my $part_nr = 0;
+ my $inc_size = 0;
+
+ foreach my $filename (@tar_parts) {
- foreach my $tarfilename (@tar_parts) {
+ next if $filename eq "$host.$num.md5";
- print "\n\t- $
tar
filename";
+ print "\n\t- $filename";
- my $path = "$Conf{ArchiveDest}/$tarfilename";
+ my $path = "$Conf{ArchiveDest}/$filename";
+ $path =~ s{//+}{/}g;
my $size = (stat( $path ))[7] || die "can't stat $path: $!";
my $size = (stat( $path ))[7] || die "can't stat $path: $!";
@@
-272,7
+276,7
@@
sub check_archive {
$tar_size_inarc += $entry->size;
if ($tar_size_inarc > $Conf{MaxArchiveFileSize}) {
$tar_size_inarc += $entry->size;
if ($tar_size_inarc > $Conf{MaxArchiveFileSize}) {
- print ", part $
tar
filename is too big $tar_size_inarc > $Conf{MaxArchiveFileSize}\n";
+ print ", part $filename is too big $tar_size_inarc > $Conf{MaxArchiveFileSize}\n";
return 0;
}
return 0;
}
@@
-296,22
+300,28
@@
sub check_archive {
=cut
# FIXME
=cut
# FIXME
- my $tar_size =
$path =~ m/.gz/ ? get_gzip_size( $path ) : -s $path
;
+ my $tar_size =
get_gzip_size( $path )
;
#
# finally, check if backup_parts table in database is valid
#
#
# finally, check if backup_parts table in database is valid
#
- my $md5 = $md5sum->{$
path} || die "no md5sum for $path"
;
+ my $md5 = $md5sum->{$
filename} || die "no md5sum for $filename in ",dump($md5sum)
;
my $items = 1;
my $items = 1;
+ $part_nr++;
- check_part($host,
undef,
$num, $part_nr, $tar_size, $size, $md5, $items);
+ check_part($host, $num, $part_nr, $tar_size, $size, $md5, $items);
- $part_nr++;
+ # round increment size to 2k block size
+ $inc_size += int(($size + 2048) / 2048);
}
}
- # short-cut and exit;
- return $same unless($same);
+ $sth_inc_size->execute(
+ $inc_size,
+ $part_nr,
+ get_backup_id($host, $num),
+ );
+ $dbh->commit;
@tar_files = sort @tar_files;
print "\n\t",($#tar_files + 1), " tar files";
@tar_files = sort @tar_files;
print "\n\t",($#tar_files + 1), " tar files";
@@
-337,6
+347,8
@@
sub check_archive {
@db_files = sort @db_files;
@db_files = sort @db_files;
+ my $same = 1;
+
if ($#tar_files != $#db_files) {
$same = 0;
print " NUMBER";
if ($#tar_files != $#db_files) {
$same = 0;
print " NUMBER";
@@
-359,6
+371,17
@@
sub check_archive {
#----- main
#----- main
+foreach ( 0 .. $#{ $args->{HostList} } ) {
+
+ my $host = $args->{'HostList'}->[$_];
+ my $num = $args->{'BackupList'}->[$_];
+
+ check_archive $host => $num;
+
+}
+
+exit;
+
my $sth = $dbh->prepare( qq{
select
my $sth = $dbh->prepare( qq{
select