automatic update of HyperEstraier index after each increment,

added -i flag to force re-scan of all files and insert missing
ones into index

git-svn-id: svn+ssh://llin/home/dpavlin/private/svn/BackupPC/trunk@89 8392b6e1-25fa-0310-8288-cc32f8e212ea

diff --git a/bin/BackupPC_updatedb b/bin/BackupPC_updatedb
index 861e495..39bd9fc 100755 (executable)
--- a/bin/BackupPC_updatedb
+++ b/bin/BackupPC_updatedb
@@ -41,6 +41,9 @@ my $beenThere = {};
 my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n";
 my $user = $Conf{SearchUser} || '';
 my $index_path = $Conf{HyperEstraierIndex};
+$index_path = $TopDir . '/' . $index_path;
+$index_path =~ s#//#/#g;
+
 
 my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });
 
@@ -80,15 +83,26 @@ sub curr_time {
        return strftime($t_fmt,localtime());
 }
 
-#---- /subs ----
+my $hest_db;
 
-## update index ##
-if ($opt{i}) {
+sub hest_update {
+
+       my ($host_id, $share_id, $num) = @_;
 
        print curr_time," updating HyperEstraier: files";
 
        my $t = time();
-       
+
+       my $where = '';
+       if ($host_id && $share_id && $num) {
+               $where = qq{
+               WHERE
+                       hosts.id = ? AND
+                       shares.id = ? AND
+                       files.backupnum = ?
+               };
+       }
+
        my $sth = $dbh->prepare(qq{
                SELECT
                        files.id                        AS fid,
@@ -107,12 +121,18 @@ if ($opt{i}) {
                        INNER JOIN shares       ON files.shareID=shares.ID
                        INNER JOIN hosts        ON hosts.ID = shares.hostID
                        INNER JOIN backups      ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID
+               $where
        });
 
-       $sth->execute();
+       $sth->execute(@_);
        my $results = $sth->rows;
 
-       my $dot = int($results / 15);
+       if ($results == 0) {
+               print " no files\n";
+               return;
+       }
+
+       my $dot = int($results / 15) || 1;
 
        print " $results ($dot/#)";
 
@@ -123,25 +143,35 @@ if ($opt{i}) {
                return $iso;
        }
 
-       my $i = 0;
        my $max = int($results / $dot);
 
-       $index_path = $TopDir . '/' . $index_path;
-       $index_path =~ s#//#/#g;
-
        print " index $index_path...";
        use HyperEstraier;
        my $db = HyperEstraier::Database->new();
+
+#      unless ($hest_db) {
+#              print " open reader";
+#              $hest_db = HyperEstraier::Database->new();
+#
+#      }
+
+
        $db->open($index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT);
 
+       my $added = 0;
 
        while (my $row = $sth->fetchrow_hashref()) {
 
+               my $fid = $row->{'fid'} || die "no fid?";
+               my $uri = 'file:///' . $fid;
+
+               next if ($db->uri_to_id($uri));
+
                # create a document object 
                my $doc = HyperEstraier::Document->new;
 
                # add attributes to the document object 
-               $doc->add_attr('@uri', 'file:///' . $row->{'fid'});
+               $doc->add_attr('@uri', $uri);
 
                foreach my $c (@{ $sth->{NAME} }) {
                        $doc->add_attr($c, $row->{$c}) if ($row->{$c});
@@ -160,30 +190,41 @@ if ($opt{i}) {
                # register the document object to the database
                $db->put_doc($doc, $HyperEstraier::Database::PDCLEAN);
 
-               $i++;
-               if ($i % $dot == 0) {
+               $added++;
+               if ($added % $dot == 0) {
                        print "$max ";
                        $max--;
                }
 
        }
 
-       print "sync";
+       print "sync $added new files";
        $db->sync();
        print " close";
        $db->close();
 
        my $dur = (time() - $t) || 1;
-       printf(" [%.2f/s dur: %s]\n",
+       printf(" [%.2f/s new %.2f/s dur: %s]\n",
                ( $results / $dur ),
+               ( $added / $dur ),
                fmt_time($dur)
        );
-
-       exit;
 }
 
-###################################create tables############################3
+#---- /subs ----
+
+
+## update index ##
+if ($opt{i} || ($index_path && ! -e $index_path)) {
+       # update all
+       print "force update of HyperEstraier index ";
+       print "importing existing data" unless (-e $index_path);
+       print "by -i flag" if ($opt{i});
+       print "\n";
+       hest_update();
+}
 
+## create tables ##
 if ($opt{c}) {
        sub do_index {
                my $index = shift || return;
@@ -273,6 +314,7 @@ if ($opt{c}) {
 
 }
 
+## delete data before inseting ##
 if ($opt{d}) {
        print "deleting ";
        foreach my $table (qw(files dvds backups shares hosts)) {
@@ -284,7 +326,7 @@ if ($opt{d}) {
        $dbh->commit;
 }
 
-#################################INSERT VALUES#############################
+## insert new values ##
 
 # get hosts
 $hosts = $bpc->HostInfoRead();
@@ -396,6 +438,8 @@ foreach my $host_key (keys %{$hosts}) {
                                ( ($f+$d) / $dur ),
                                fmt_time($dur)
                        );
+
+                       hest_update($hostID, $shareID, $backupNum);
                }
 
        }