projects / BackupPC.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
r9171@llin: dpavlin | 2006-01-31 23:04:30 +0100
[BackupPC.git] / bin / BackupPC_updatedb
diff --git a/bin/BackupPC_updatedb b/bin/BackupPC_updatedb
index 48dcffb..3657d28 100755 (executable)
--- a/bin/BackupPC_updatedb
+++ b/bin/BackupPC_updatedb
@@ -11,15 +11,26 @@ use Getopt::Std;
 use Time::HiRes qw/time/;
 use File::Pid;
 use POSIX qw/strftime/;
 use Time::HiRes qw/time/;
 use File::Pid;
 use POSIX qw/strftime/;
+use BackupPC::SearchLib;
+use Cwd qw/abs_path/;
 
 use constant BPC_FTYPE_DIR => 5;
 
 use constant BPC_FTYPE_DIR => 5;
+use constant EST_CHUNK => 4096;
+
+# daylight saving time change offset for 1h
+my $dst_offset = 60 * 60;
 
 my $debug = 0;
 $|=1;
 
 my $start_t = time();
 
 
 my $debug = 0;
 $|=1;
 
 my $start_t = time();
 
-my $pidfile = new File::Pid;
+my $pid_path = abs_path($0);
+$pid_path =~ s/\W+/_/g;
+
+my $pidfile = new File::Pid({
+       file => "/tmp/$pid_path",
+});
 
 if (my $pid = $pidfile->running ) {
        die "$0 already running: $pid\n";
 
 if (my $pid = $pidfile->running ) {
        die "$0 already running: $pid\n";
@@ -27,8 +38,8 @@ if (my $pid = $pidfile->running ) {
        $pidfile->remove;
        $pidfile = new File::Pid;
 }
        $pidfile->remove;
        $pidfile = new File::Pid;
 }
-$pidfile->write;
 print STDERR "$0 using pid ",$pidfile->pid," file ",$pidfile->file,"\n";
 print STDERR "$0 using pid ",$pidfile->pid," file ",$pidfile->file,"\n";
+$pidfile->write;
 
 my $t_fmt = '%Y-%m-%d %H:%M:%S';
 
 
 my $t_fmt = '%Y-%m-%d %H:%M:%S';
 
@@ -41,112 +52,401 @@ my $beenThere = {};
 my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n";
 my $user = $Conf{SearchUser} || '';
 
 my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n";
 my $user = $Conf{SearchUser} || '';
 
+my $index_node_url = $Conf{HyperEstraierIndex};
+
 my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });
 
 my %opt;
 
 my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });
 
 my %opt;
 
-if ( !getopts("cdm:v:", \%opt ) ) {
+if ( !getopts("cdm:v:ijfq", \%opt ) ) {
        print STDERR <<EOF;
        print STDERR <<EOF;
-usage: $0 [-c|-d] [-m num] [-v|-v level]
+usage: $0 [-c|-d] [-m num] [-v|-v level] [-i|-j|-f]
 
 Options:
        -c      create database on first use
        -d      delete database before import
        -m num  import just num increments for one host
        -v num  set verbosity (debug) level (default $debug)
 
 Options:
        -c      create database on first use
        -d      delete database before import
        -m num  import just num increments for one host
        -v num  set verbosity (debug) level (default $debug)
+       -i      update Hyper Estraier full text index
+       -j      update full text, don't check existing files
+       -f      don't do anything with full text index
+       -q      be quiet for hosts without changes
+
+Option -j is variation on -i. It will allow faster initial creation
+of full-text index from existing database.
+
+Option -f will create database which is out of sync with full text index. You
+will have to re-run $0 with -i to fix it.
+
 EOF
        exit 1;
 }
 
 EOF
        exit 1;
 }
 
-###################################create tables############################3
+if ($opt{v}) {
+       print "Debug level at $opt{v}\n";
+       $debug = $opt{v};
+} elsif ($opt{f}) {
+       print "WARNING: disabling full-text index update. You need to re-run $0 -j !\n";
+       $index_node_url = undef;
+}
+
+#---- subs ----
 
 
+sub fmt_time {
+       my $t = shift || return;
+       my $out = "";
+       my ($ss,$mm,$hh) = gmtime($t);
+       $out .= "${hh}h" if ($hh);
+       $out .= sprintf("%02d:%02d", $mm,$ss);
+       return $out;
+}
+
+sub curr_time {
+       return strftime($t_fmt,localtime());
+}
+
+my $hest_node;
+
+sub hest_update {
+
+       my ($host_id, $share_id, $num) = @_;
+
+       my $skip_check = $opt{j} && print STDERR "Skipping check for existing files -- this should be used only with initital import\n";
+
+       unless ($index_node_url && $index_node_url =~ m#^http://#) {
+               print STDERR "HyperEstraier support not enabled or index node invalid\n" if ($debug);
+               $index_node_url = 0;
+               return;
+       }
+
+       print curr_time," updating Hyper Estraier:";
+
+       my $t = time();
+
+       my $offset = 0;
+       my $added = 0;
+
+       if ($index_node_url) {
+               print " opening index $index_node_url";
+               $hest_node ||= Search::Estraier::Node->new(
+                       url => $index_node_url,
+                       user => 'admin',
+                       passwd => 'admin',
+                       croak_on_error => 1,
+               );
+               print " via node URL";
+       }
+
+       my $results = 0;
+
+       do {
+
+               my $where = '';
+               my @data;
+               if (defined($host_id) && defined($share_id) && defined($num)) {
+                       $where = qq{
+                       WHERE
+                               hosts.id = ? AND
+                               shares.id = ? AND
+                               files.backupnum = ?
+                       };
+                       @data = ( $host_id, $share_id, $num );
+               }
+
+               my $limit = sprintf('LIMIT '.EST_CHUNK.' OFFSET %d', $offset);
+
+               my $sth = $dbh->prepare(qq{
+                       SELECT
+                               files.id                        AS fid,
+                               hosts.name                      AS hname,
+                               shares.name                     AS sname,
+                               -- shares.share                 AS sharename,
+                               files.backupnum                 AS backupnum,
+                               -- files.name                   AS filename,
+                               files.path                      AS filepath,
+                               files.date                      AS date,
+                               files.type                      AS type,
+                               files.size                      AS size,
+                               files.shareid                   AS shareid,
+                               backups.date                    AS backup_date
+                       FROM files 
+                               INNER JOIN shares       ON files.shareID=shares.ID
+                               INNER JOIN hosts        ON hosts.ID = shares.hostID
+                               INNER JOIN backups      ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID
+                       $where
+                       $limit
+               });
+
+               $sth->execute(@data);
+               $results = $sth->rows;
+
+               if ($results == 0) {
+                       print " - no new files\n";
+                       return;
+               } else {
+                       print "...";
+               }
+
+               sub fmt_date {
+                       my $t = shift || return;
+                       my $iso = BackupPC::Lib::timeStamp($t);
+                       $iso =~ s/\s/T/;
+                       return $iso;
+               }
+
+               while (my $row = $sth->fetchrow_hashref()) {
+
+                       my $uri = $row->{hname} . ':' . $row->{sname} . '#' . $row->{backupnum} . ' ' . $row->{filepath};
+                       if (! $skip_check && $hest_node) {
+                               my $id = $hest_node->uri_to_id($uri);
+                               next if ($id && $id == -1);
+                       }
+
+                       # create a document object 
+                       my $doc = Search::Estraier::Document->new;
+
+                       # add attributes to the document object 
+                       $doc->add_attr('@uri', $uri);
+
+                       foreach my $c (@{ $sth->{NAME} }) {
+                               print STDERR "attr $c = $row->{$c}\n" if ($debug > 2);
+                               $doc->add_attr($c, $row->{$c}) if (defined($row->{$c}));
+                       }
+
+                       #$doc->add_attr('@cdate', fmt_date($row->{'date'}));
+
+                       # add the body text to the document object 
+                       my $path = $row->{'filepath'};
+                       $doc->add_text($path);
+                       $path =~ s/(.)/$1 /g;
+                       $doc->add_hidden_text($path);
+
+                       print STDERR $doc->dump_draft,"\n" if ($debug > 1);
+
+                       # register the document object to the database
+                       $hest_node->put_doc($doc) if ($hest_node);
+
+                       $added++;
+               }
+
+               print "$added";
+
+               $offset += EST_CHUNK;
+
+       } while ($results == EST_CHUNK);
+
+       my $dur = (time() - $t) || 1;
+       printf(" [%.2f/s dur: %s]\n",
+               ( $added / $dur ),
+               fmt_time($dur)
+       );
+}
+
+#---- /subs ----
+
+
+## update index ##
+if ( ( $opt{i} || $opt{j} ) && !$opt{c} ) {
+       # update all
+       print "force update of Hyper Estraier index ";
+       print "by -i flag" if ($opt{i});
+       print "by -j flag" if ($opt{j});
+       print "\n";
+       hest_update();
+}
+
+## create tables ##
 if ($opt{c}) {
        sub do_index {
                my $index = shift || return;
 if ($opt{c}) {
        sub do_index {
                my $index = shift || return;
-               my ($table,$col,$unique) = split(/_/, $index);
+               my ($table,$col,$unique) = split(/:/, $index);
                $unique ||= '';
                $unique ||= '';
+               $index =~ s/\W+/_/g;
+               print "$index on $table($col)" . ( $unique ? "u" : "" ) . " ";
                $dbh->do(qq{ create $unique index $index on $table($col) });
        }
 
        print "creating tables...\n";
                $dbh->do(qq{ create $unique index $index on $table($col) });
        }
 
        print "creating tables...\n";
-      
-       $dbh->do(qq{
+
+       $dbh->do( qq{
                create table hosts (
                        ID      SERIAL          PRIMARY KEY,
                        name    VARCHAR(30)     NOT NULL,
                        IP      VARCHAR(15)
                );            
                create table hosts (
                        ID      SERIAL          PRIMARY KEY,
                        name    VARCHAR(30)     NOT NULL,
                        IP      VARCHAR(15)
                );            
-       });
-             
-       $dbh->do(qq{
+
                create table shares (
                        ID      SERIAL          PRIMARY KEY,
                        hostID  INTEGER         NOT NULL references hosts(id),
                        name    VARCHAR(30)     NOT NULL,
                create table shares (
                        ID      SERIAL          PRIMARY KEY,
                        hostID  INTEGER         NOT NULL references hosts(id),
                        name    VARCHAR(30)     NOT NULL,
-                       share   VARCHAR(200)    NOT NULL,
-                       localpath VARCHAR(200)      
+                       share   VARCHAR(200)    NOT NULL
                );            
                );            
-       });
-       
-       $dbh->do(qq{
+
+               create table dvds (
+                       ID      SERIAL          PRIMARY KEY, 
+                       num     INTEGER         NOT NULL,
+                       name    VARCHAR(255)    NOT NULL,
+                       mjesto  VARCHAR(255)
+               );
+
                create table backups (
                create table backups (
+                       id      serial,
                        hostID  INTEGER         NOT NULL references hosts(id),
                        num     INTEGER         NOT NULL,
                        date    integer         NOT NULL, 
                        type    CHAR(4)         not null,
                        hostID  INTEGER         NOT NULL references hosts(id),
                        num     INTEGER         NOT NULL,
                        date    integer         NOT NULL, 
                        type    CHAR(4)         not null,
-                       PRIMARY KEY(hostID, num) 
+                       shareID integer         not null references shares(id),
+                       size    bigint          not null,
+                       inc_size bigint         not null default -1,
+                       inc_deleted boolean     default false,
+                       parts   integer         not null default 0,
+                       PRIMARY KEY(id)
                );            
                );            
-       });
 
 
-       do_index('backups_num_unique');
+               create table files (
+                       ID              SERIAL,
+                       shareID         INTEGER NOT NULL references shares(id),
+                       backupNum       INTEGER NOT NULL,
+                       name            VARCHAR(255) NOT NULL,
+                       path            VARCHAR(255) NOT NULL,
+                       date            integer NOT NULL,
+                       type            INTEGER NOT NULL,
+                       size            bigint  NOT NULL,
+                       primary key(id)
+               );
 
 
-       $dbh->do(qq{
-               create table dvds (
-                       ID      SERIAL          PRIMARY KEY, 
-                       num     INTEGER         NOT NULL,
-                       name    VARCHAR(255)    NOT NULL,
-                       mjesto  VARCHAR(255)
+               create table archive (
+                       id              serial,
+                       dvd_nr          int not null,
+                       total_size      bigint default -1,
+                       note            text,
+                       username        varchar(20) not null,
+                       date            timestamp default now(),
+                       primary key(id)
+               );      
+
+               create table archive_backup (
+                       archive_id      int not null references archive(id) on delete cascade,
+                       backup_id       int not null references backups(id),
+                       primary key(archive_id, backup_id)
                );
                );
-       });
 
 
-       $dbh->do(qq{     
-               create table files (
-                       ID      SERIAL          PRIMARY KEY,  
-                       shareID INTEGER         NOT NULL references shares(id),
-                       backupNum  INTEGER      NOT NULL references backups(num),
-                       name       VARCHAR(255) NOT NULL,
-                       path       VARCHAR(255) NOT NULL,
-                       fullpath   VARCHAR(255) NOT NULL,
-                       date       integer      NOT NULL,
-                       type       INTEGER      NOT NULL,
-                       size       INTEGER      NOT NULL,
-                       dvdid      INTEGER      references dvds(id)     
+               create table archive_burned (
+                       archive_id      int references archive(id),
+                       date            timestamp default now(),
+                       part            int not null default 1,
+                       copy            int not null default 1,
+                       iso_size bigint default -1
+               );
+
+               create table backup_parts (
+                       id serial,
+                       backup_id int references backups(id),
+                       part_nr int not null check (part_nr > 0),
+                       tar_size bigint not null check (tar_size > 0),
+                       size bigint not null check (size > 0),
+                       md5 text not null,
+                       items int not null check (items > 0),
+                       date timestamp default now(),
+                       primary key(id)
                );
        });
 
                );
        });
 
-       print "creating indexes:";
+       print "creating indexes: ";
 
        foreach my $index (qw(
 
        foreach my $index (qw(
-               hosts_name
-               backups_hostID
-               backups_num
-               shares_hostID
-               shares_name
-               files_shareID
-               files_path
-               files_name
-               files_date
-               files_size
+               hosts:name
+               backups:hostID
+               backups:num
+               backups:shareID
+               shares:hostID
+               shares:name
+               files:shareID
+               files:path
+               files:name
+               files:date
+               files:size
+               archive:dvd_nr
+               archive_burned:archive_id
+               backup_parts:backup_id,part_nr
        )) {
        )) {
-               print " $index";
                do_index($index);
        }
                do_index($index);
        }
+
+       print " creating sequence: ";
+       foreach my $seq (qw/dvd_nr/) {
+               print "$seq ";
+               $dbh->do( qq{ CREATE SEQUENCE $seq } );
+       }
+
+       print " creating triggers ";
+       $dbh->do( <<__END_OF_TRIGGER__ );
+
+create or replace function backup_parts_check() returns trigger as '
+declare
+       b_parts integer;
+       b_counted integer;
+       b_id    integer;
+begin
+       -- raise notice ''old/new parts %/% backup_id %/%'', old.parts, new.parts, old.id, new.id;
+       if (TG_OP=''UPDATE'') then
+               b_id := new.id;
+               b_parts := new.parts;
+       elsif (TG_OP = ''INSERT'') then
+               b_id := new.id;
+               b_parts := new.parts;
+       end if;
+       b_counted := (select count(*) from backup_parts where backup_id = b_id);
+       -- raise notice ''backup % parts %'', b_id, b_parts;
+       if ( b_parts != b_counted ) then
+               raise exception ''Update of backup % aborted, requested % parts and there are really % parts'', b_id, b_parts, b_counted;
+       end if;
+       return null;
+end;
+' language plpgsql;
+
+create trigger do_backup_parts_check
+       after insert or update or delete on backups
+       for each row execute procedure backup_parts_check();
+
+create or replace function backup_backup_parts_check() returns trigger as '
+declare
+       b_id            integer;
+       my_part_nr      integer;
+       calc_part       integer;
+begin
+       if (TG_OP = ''INSERT'') then
+               -- raise notice ''trigger: % backup_id %'', TG_OP, new.backup_id;
+               b_id = new.backup_id;
+               my_part_nr = new.part_nr;
+               execute ''update backups set parts = parts + 1 where id = '' || b_id;
+       elsif (TG_OP = ''DELETE'') then
+               -- raise notice ''trigger: % backup_id %'', TG_OP, old.backup_id;
+               b_id = old.backup_id;
+               my_part_nr = old.part_nr;
+               execute ''update backups set parts = parts - 1 where id = '' || b_id;
+       end if;
+       calc_part := (select count(part_nr) from backup_parts where backup_id = b_id);
+       if ( my_part_nr != calc_part ) then
+               raise exception ''Update of backup_parts with backup_id % aborted, requested part_nr is % and calulated next is %'', b_id, my_part_nr, calc_part;
+       end if;
+       return null;
+end;
+' language plpgsql;
+
+create trigger do_backup_backup_parts_check
+       after insert or update or delete on backup_parts
+       for each row execute procedure backup_backup_parts_check();
+
+__END_OF_TRIGGER__
+
        print "...\n";
 
        $dbh->commit;
 
 }
 
        print "...\n";
 
        $dbh->commit;
 
 }
 
+## delete data before inseting ##
 if ($opt{d}) {
        print "deleting ";
        foreach my $table (qw(files dvds backups shares hosts)) {
 if ($opt{d}) {
        print "deleting ";
        foreach my $table (qw(files dvds backups shares hosts)) {
@@ -158,12 +458,7 @@ if ($opt{d}) {
        $dbh->commit;
 }
 
        $dbh->commit;
 }
 
-if ($opt{v}) {
-       print "Debug level at $opt{v}\n";
-       $debug = $opt{v};
-}
-
-#################################INSERT VALUES#############################
+## insert new values ##
 
 # get hosts
 $hosts = $bpc->HostInfoRead();
 
 # get hosts
 $hosts = $bpc->HostInfoRead();
@@ -180,33 +475,32 @@ $sth->{hosts_by_name} = $dbh->prepare(qq{
 SELECT ID FROM hosts WHERE name=?
 });
 
 SELECT ID FROM hosts WHERE name=?
 });
 
-$sth->{backups_broj} = $dbh->prepare(qq{
+$sth->{backups_count} = $dbh->prepare(qq{
 SELECT COUNT(*)
 FROM backups
 SELECT COUNT(*)
 FROM backups
-WHERE hostID=? AND num=?
+WHERE hostID=? AND num=? AND shareid=?
 });
 
 $sth->{insert_backups} = $dbh->prepare(qq{
 });
 
 $sth->{insert_backups} = $dbh->prepare(qq{
-INSERT INTO backups (hostID, num, date, type)
-VALUES (?,?,?,?)
+INSERT INTO backups (hostID, num, date, type, shareid, size)
+VALUES (?,?,?,?,?,-1)
+});
+
+$sth->{update_backups_size} = $dbh->prepare(qq{
+UPDATE backups SET size = ?
+WHERE hostID = ? and num = ? and date = ? and type =? and shareid = ?
 });
 
 $sth->{insert_files} = $dbh->prepare(qq{
 INSERT INTO files
 });
 
 $sth->{insert_files} = $dbh->prepare(qq{
 INSERT INTO files
-       (shareID, backupNum, name, path, fullpath, date, type, size)
-       VALUES (?,?,?,?,?,?,?,?)
+       (shareID, backupNum, name, path, date, type, size)
+       VALUES (?,?,?,?,?,?,?)
 });
 
 });
 
-sub fmt_time {
-       my $t = shift || return;
-       my $out = "";
-       my ($ss,$mm,$hh) = gmtime($t);
-       $out .= "${hh}h" if ($hh);
-       $out .= sprintf("%02d:%02d", $mm,$ss);
-       return $out;
-}
+my @hosts = keys %{$hosts};
+my $host_nr = 0;
 
 
-foreach my $host_key (keys %{$hosts}) {
+foreach my $host_key (@hosts) {
 
        my $hostname = $hosts->{$host_key}->{'host'} || die "can't find host for $host_key";
 
 
        my $hostname = $hosts->{$host_key}->{'host'} || die "can't find host for $host_key";
 
@@ -221,13 +515,21 @@ foreach my $host_key (keys %{$hosts}) {
                $hostID = $dbh->last_insert_id(undef,undef,'hosts',undef);
        }
 
                $hostID = $dbh->last_insert_id(undef,undef,'hosts',undef);
        }
 
-       print("host ".$hosts->{$host_key}->{'host'}.": ");
- 
+       $host_nr++;
        # get backups for a host
        my @backups = $bpc->BackupInfoRead($hostname);
        # get backups for a host
        my @backups = $bpc->BackupInfoRead($hostname);
-       print scalar @backups, " increments\n";
-
+       my $incs = scalar @backups;
+
+       my $host_header = sprintf("host %s [%d/%d]: %d increments\n",
+               $hosts->{$host_key}->{'host'},
+               $host_nr,
+               ($#hosts + 1),
+               $incs
+       );
+       print $host_header unless ($opt{q});
+ 
        my $inc_nr = 0;
        my $inc_nr = 0;
+       $beenThere = {};
 
        foreach my $backup (@backups) {
 
 
        foreach my $backup (@backups) {
 
@@ -237,39 +539,76 @@ foreach my $host_key (keys %{$hosts}) {
                my $backupNum = $backup->{'num'};
                my @backupShares = ();
 
                my $backupNum = $backup->{'num'};
                my @backupShares = ();
 
-               print $hosts->{$host_key}->{'host'},
-                       "\t#$backupNum\t", $backup->{type} || '?', " ",
-                       $backup->{nFilesNew} || '?', "/", $backup->{nFiles} || '?',
-                       " files\n";
-
-               $sth->{backups_broj}->execute($hostID, $backupNum);
-               my ($broj) = $sth->{backups_broj}->fetchrow_array();
-               next if ($broj > 0);
-
-               $sth->{insert_backups}->execute(
-                       $hostID,
-                       $backupNum,
-                       $backup->{'endTime'},
-                       $backup->{'type'}
+               my $share_header = sprintf("%-10s %2d/%-2d #%-2d %s %5s/%5s files (date: %s dur: %s)\n", 
+                       $hosts->{$host_key}->{'host'},
+                       $inc_nr, $incs, $backupNum, 
+                       $backup->{type} || '?',
+                       $backup->{nFilesNew} || '?', $backup->{nFiles} || '?',
+                       strftime($t_fmt,localtime($backup->{startTime})),
+                       fmt_time($backup->{endTime} - $backup->{startTime})
                );
                );
-               $dbh->commit();
+               print $share_header unless ($opt{q});
 
                my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1);
                foreach my $share ($files->shareList($backupNum)) {
 
                        my $t = time();
 
 
                my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1);
                foreach my $share ($files->shareList($backupNum)) {
 
                        my $t = time();
 
-                       print strftime($t_fmt,localtime())," ", $share;
                        $shareID = getShareID($share, $hostID, $hostname);
                
                        $shareID = getShareID($share, $hostID, $hostname);
                
-                       my ($f, $nf, $d, $nd) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);
+                       $sth->{backups_count}->execute($hostID, $backupNum, $shareID);
+                       my ($count) = $sth->{backups_count}->fetchrow_array();
+                       # skip if allready in database!
+                       next if ($count > 0);
+
+                       # dump host and share header for -q
+                       if ($opt{q}) {
+                               if ($host_header) {
+                                       print $host_header;
+                                       $host_header = undef;
+                               }
+                               print $share_header;
+                       }
+
+                       # dump some log
+                       print curr_time," ", $share;
+
+                       $sth->{insert_backups}->execute(
+                               $hostID,
+                               $backupNum,
+                               $backup->{'endTime'},
+                               substr($backup->{'type'},0,4),
+                               $shareID,
+                       );
+
+                       my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);
+
+                       eval {
+                               $sth->{update_backups_size}->execute(
+                                       $size,
+                                       $hostID,
+                                       $backupNum,
+                                       $backup->{'endTime'},
+                                       substr($backup->{'type'},0,4),
+                                       $shareID,
+                               );
+                               print " commit";
+                               $dbh->commit();
+                       };
+                       if ($@) {
+                               print " rollback";
+                               $dbh->rollback();
+                       }
+
                        my $dur = (time() - $t) || 1;
                        my $dur = (time() - $t) || 1;
-                       printf(" %d/%d files %d/%d dirs [%.2f/s dur: %s]\n",
+                       printf(" %d/%d files %d/%d dirs %0.2f MB [%.2f/s dur: %s]\n",
                                $nf, $f, $nd, $d,
                                $nf, $f, $nd, $d,
+                               ($size / 1024 / 1024),
                                ( ($f+$d) / $dur ),
                                fmt_time($dur)
                        );
                                ( ($f+$d) / $dur ),
                                fmt_time($dur)
                        );
-                       $dbh->commit();
+
+                       hest_update($hostID, $shareID, $backupNum) if ($nf + $nd > 0);
                }
 
        }
                }
 
        }
@@ -298,14 +637,14 @@ sub getShareID() {
 
        $sth->{insert_share} ||= $dbh->prepare(qq{
                INSERT INTO shares 
 
        $sth->{insert_share} ||= $dbh->prepare(qq{
                INSERT INTO shares 
-                       (hostID,name,share,localpath) 
-               VALUES (?,?,?,?)
+                       (hostID,name,share) 
+               VALUES (?,?,?)
        });
 
        my $drop_down = $hostname . '/' . $share;
        $drop_down =~ s#//+#/#g;
 
        });
 
        my $drop_down = $hostname . '/' . $share;
        $drop_down =~ s#//+#/#g;
 
-       $sth->{insert_share}->execute($hostID,$share, $drop_down ,undef);
+       $sth->{insert_share}->execute($hostID,$share, $drop_down);
        return $dbh->last_insert_id(undef,undef,'shares',undef);
 }
 
        return $dbh->last_insert_id(undef,undef,'shares',undef);
 }
 
@@ -314,7 +653,7 @@ sub found_in_db {
        my @data = @_;
        shift @data;
 
        my @data = @_;
        shift @data;
 
-       my ($key, $shareID,undef,$name,$path,undef,$date,undef,$size) = @_;
+       my ($key, $shareID,undef,$name,$path,$date,undef,$size) = @_;
 
        return $beenThere->{$key} if (defined($beenThere->{$key}));
 
 
        return $beenThere->{$key} if (defined($beenThere->{$key}));
 
@@ -322,15 +661,15 @@ sub found_in_db {
                SELECT 1 FROM files
                WHERE shareID = ? and
                        path = ? and 
                SELECT 1 FROM files
                WHERE shareID = ? and
                        path = ? and 
-                       name = ? and
-                       date = ? and
-                       size = ?
+                       size = ? and
+                       ( date = ? or date = ? or date = ? )
+               LIMIT 1
        });
 
        });
 
-       my @param = ($shareID,$path,$name,$date,$size);
+       my @param = ($shareID,$path,$size,$date, $date-$dst_offset, $date+$dst_offset);
        $sth->{file_in_db}->execute(@param);
        my $rows = $sth->{file_in_db}->rows;
        $sth->{file_in_db}->execute(@param);
        my $rows = $sth->{file_in_db}->rows;
-       print STDERR "## found_in_db ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);
+       print STDERR "## found_in_db($shareID,$path,$date,$size) ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);
 
        $beenThere->{$key}++;
 
 
        $beenThere->{$key}++;
 
@@ -348,7 +687,7 @@ sub recurseDir($$$$$$$$) {
 
        print STDERR "\nrecurse($hostname,$backupNum,$share,$dir,$shareID)\n" if ($debug >= 1);
 
 
        print STDERR "\nrecurse($hostname,$backupNum,$share,$dir,$shareID)\n" if ($debug >= 1);
 
-       my ($nr_files, $new_files, $nr_dirs, $new_dirs) = (0,0,0,0);
+       my ($nr_files, $new_files, $nr_dirs, $new_dirs, $size) = (0,0,0,0,0);
 
        { # scope
                my @stack;
 
        { # scope
                my @stack;
@@ -358,13 +697,12 @@ sub recurseDir($$$$$$$$) {
 
                # first, add all the entries in current directory
                foreach my $path_key (keys %{$filesInBackup}) {
 
                # first, add all the entries in current directory
                foreach my $path_key (keys %{$filesInBackup}) {
+                       print STDERR "# file ",Dumper($filesInBackup->{$path_key}),"\n" if ($debug >= 3);
                        my @data = (
                                $shareID,
                                $backupNum,
                                $path_key,
                                $filesInBackup->{$path_key}->{'relPath'},
                        my @data = (
                                $shareID,
                                $backupNum,
                                $path_key,
                                $filesInBackup->{$path_key}->{'relPath'},
-                               $filesInBackup->{$path_key}->{'fullPath'},
-       #                       $filesInBackup->{$path_key}->{'sharePathM'},
                                $filesInBackup->{$path_key}->{'mtime'},
                                $filesInBackup->{$path_key}->{'type'},
                                $filesInBackup->{$path_key}->{'size'}
                                $filesInBackup->{$path_key}->{'mtime'},
                                $filesInBackup->{$path_key}->{'type'},
                                $filesInBackup->{$path_key}->{'size'}
@@ -378,17 +716,39 @@ sub recurseDir($$$$$$$$) {
                                $filesInBackup->{$path_key}->{'size'}
                        ));
 
                                $filesInBackup->{$path_key}->{'size'}
                        ));
 
+                       my $key_dst_prev = join(" ", (
+                               $shareID,
+                               $dir,
+                               $path_key,
+                               $filesInBackup->{$path_key}->{'mtime'} - $dst_offset,
+                               $filesInBackup->{$path_key}->{'size'}
+                       ));
+
+                       my $key_dst_next = join(" ", (
+                               $shareID,
+                               $dir,
+                               $path_key,
+                               $filesInBackup->{$path_key}->{'mtime'} + $dst_offset,
+                               $filesInBackup->{$path_key}->{'size'}
+                       ));
 
 
-                       if (! defined($beenThere->{$key}) && ! found_in_db($key, @data)) {
+                       my $found;
+                       if (
+                               ! defined($beenThere->{$key}) &&
+                               ! defined($beenThere->{$key_dst_prev}) &&
+                               ! defined($beenThere->{$key_dst_next}) &&
+                               ! ($found = found_in_db($key, @data))
+                       ) {
                                print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2);
 
                                if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
                                print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2);
 
                                if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
-                                       $new_dirs++;
+                                       $new_dirs++ unless ($found);
                                        print STDERR " dir\n" if ($debug >= 2);
                                } else {
                                        print STDERR " dir\n" if ($debug >= 2);
                                } else {
-                                       $new_files++;
+                                       $new_files++ unless ($found);
                                        print STDERR " file\n" if ($debug >= 2);
                                }
                                        print STDERR " file\n" if ($debug >= 2);
                                }
+                               $size += $filesInBackup->{$path_key}->{'size'} || 0;
                        }
 
                        if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
                        }
 
                        if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
@@ -413,15 +773,16 @@ sub recurseDir($$$$$$$$) {
                print STDERR "## STACK ",join(", ", @stack),"\n" if ($debug >= 2);
 
                while ( my $dir = shift @stack ) {
                print STDERR "## STACK ",join(", ", @stack),"\n" if ($debug >= 2);
 
                while ( my $dir = shift @stack ) {
-                       my ($f,$nf,$d,$nd) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID);
+                       my ($f,$nf,$d,$nd, $s) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID);
                        print STDERR "# $dir f: $f nf: $nf d: $d nd: $nd\n" if ($debug >= 1);
                        $nr_files += $f;
                        $new_files += $nf;
                        $nr_dirs += $d;
                        $new_dirs += $nd;
                        print STDERR "# $dir f: $f nf: $nf d: $d nd: $nd\n" if ($debug >= 1);
                        $nr_files += $f;
                        $new_files += $nf;
                        $nr_dirs += $d;
                        $new_dirs += $nd;
+                       $size += $s;
                }
        }
 
                }
        }
 
-       return ($nr_files, $new_files, $nr_dirs, $new_dirs);
+       return ($nr_files, $new_files, $nr_dirs, $new_dirs, $size);
 }
 
 }
 
BackupPC search extension