use File::Pid;
use POSIX qw/strftime/;
use BackupPC::SearchLib;
+use Cwd qw/abs_path/;
use constant BPC_FTYPE_DIR => 5;
use constant EST_CHUNK => 100000;
+# daylight saving time change offset for 1h
+my $dst_offset = 60 * 60;
+
my $debug = 0;
$|=1;
my $start_t = time();
-my $pidfile = new File::Pid;
+my $pid_path = abs_path($0);
+$pid_path =~ s/\W+/_/g;
+
+my $pidfile = new File::Pid({
+ file => "/tmp/$pid_path",
+});
if (my $pid = $pidfile->running ) {
die "$0 already running: $pid\n";
$pidfile->remove;
$pidfile = new File::Pid;
}
-$pidfile->write;
print STDERR "$0 using pid ",$pidfile->pid," file ",$pidfile->file,"\n";
+$pidfile->write;
my $t_fmt = '%Y-%m-%d %H:%M:%S';
my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n";
my $user = $Conf{SearchUser} || '';
-my $use_hest = $Conf{HyperEstraierIndex};
-my ($index_path, $index_node_url) = BackupPC::SearchLib::getHyperEstraier_url($use_hest);
+my $index_node_url = $Conf{HyperEstraierIndex};
my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });
my %opt;
-if ( !getopts("cdm:v:ij", \%opt ) ) {
+if ( !getopts("cdm:v:ijf", \%opt ) ) {
print STDERR <<EOF;
-usage: $0 [-c|-d] [-m num] [-v|-v level] [-i]
+usage: $0 [-c|-d] [-m num] [-v|-v level] [-i|-j|-f]
Options:
-c create database on first use
-v num set verbosity (debug) level (default $debug)
-i update Hyper Estraier full text index
-j update full text, don't check existing files
+ -f don't do anything with full text index
Option -j is variation on -i. It will allow faster initial creation
of full-text index from existing database.
+Option -f will create database which is out of sync with full text index. You
+will have to re-run $0 with -i to fix it.
+
EOF
exit 1;
}
if ($opt{v}) {
print "Debug level at $opt{v}\n";
$debug = $opt{v};
+} elsif ($opt{f}) {
+ print "WARNING: disabling full-text index update. You need to re-run $0 -j !\n";
+ $index_node_url = undef;
}
#---- subs ----
my $skip_check = $opt{j} && print STDERR "Skipping check for existing files -- this should be used only with initital import\n";
- unless ($use_hest) {
+ unless (defined($index_node_url)) {
print STDERR "HyperEstraier support not enabled in configuration\n";
+ $index_node_url = 0;
return;
}
- print curr_time," updating HyperEstraier:";
+ print curr_time," updating Hyper Estraier:";
my $t = time();
my $offset = 0;
my $added = 0;
- print " opening index $use_hest";
- if ($index_path) {
- $hest_db = HyperEstraier::Database->new();
- $hest_db->open($TopDir . $index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT);
- print " directly";
- } elsif ($index_node_url) {
- $hest_node ||= HyperEstraier::Node->new($index_node_url);
+ print " opening index $index_node_url";
+ if ($index_node_url) {
+ $hest_node ||= Search::Estraier::Node->new($index_node_url);
$hest_node->set_auth('admin', 'admin');
print " via node URL";
} else {
- die "don't know how to use HyperEstraier Index $use_hest";
+ die "don't know how to use Hyper Estraier Index $index_node_url";
}
- print " increment is " . EST_CHUNK . " files:";
my $results = 0;
my $where = '';
my @data;
- if ($host_id && $share_id && $num) {
+ if (defined($host_id) && defined($share_id) && defined($num)) {
$where = qq{
WHERE
hosts.id = ? AND
if ($results == 0) {
print " - no new files\n";
last;
+ } else {
+ print " - $results files: ";
}
sub fmt_date {
}
# create a document object
- my $doc = HyperEstraier::Document->new;
+ my $doc = Search::Estraier::Document->new;
# add attributes to the document object
$doc->add_attr('@uri', $uri);
foreach my $c (@{ $sth->{NAME} }) {
- $doc->add_attr($c, $row->{$c}) if ($row->{$c});
+ print STDERR "attr $c = $row->{$c}\n" if ($debug > 2);
+ $doc->add_attr($c, $row->{$c}) if (defined($row->{$c}));
}
#$doc->add_attr('@cdate', fmt_date($row->{'date'}));
print STDERR $doc->dump_draft,"\n" if ($debug > 1);
# register the document object to the database
- if ($hest_db) {
- $hest_db->put_doc($doc, $HyperEstraier::Database::PDCLEAN);
- } elsif ($hest_node) {
+ if ($hest_node) {
$hest_node->put_doc($doc);
} else {
die "not supported";
}
print " $added";
- $hest_db->sync() if ($index_path);
$offset += EST_CHUNK;
} while ($results == EST_CHUNK);
- if ($index_path) {
- print ", close";
- $hest_db->close();
- }
-
my $dur = (time() - $t) || 1;
printf(" [%.2f/s dur: %s]\n",
( $added / $dur ),
## update index ##
-if (($opt{i} || $opt{j} || ($index_path && ! -e $index_path)) && !$opt{c}) {
+if ( ( $opt{i} || $opt{j} ) && !$opt{c} ) {
# update all
- print "force update of HyperEstraier index ";
- print "importing existing data" unless (-e $index_path);
+ print "force update of Hyper Estraier index ";
print "by -i flag" if ($opt{i});
print "by -j flag" if ($opt{j});
print "\n";
}
print "creating tables...\n";
-
- $dbh->do(qq{
+
+ $dbh->do( qq{
create table hosts (
ID SERIAL PRIMARY KEY,
name VARCHAR(30) NOT NULL,
IP VARCHAR(15)
);
- });
-
- $dbh->do(qq{
+
create table shares (
ID SERIAL PRIMARY KEY,
hostID INTEGER NOT NULL references hosts(id),
name VARCHAR(30) NOT NULL,
share VARCHAR(200) NOT NULL
);
- });
- $dbh->do(qq{
create table dvds (
ID SERIAL PRIMARY KEY,
num INTEGER NOT NULL,
name VARCHAR(255) NOT NULL,
mjesto VARCHAR(255)
);
- });
-
- $dbh->do(qq{
+
create table backups (
id serial,
hostID INTEGER NOT NULL references hosts(id),
size bigint not null,
inc_size bigint not null default -1,
inc_deleted boolean default false,
+ parts integer not null default 1,
PRIMARY KEY(id)
);
- });
- $dbh->do(qq{
create table files (
ID SERIAL,
shareID INTEGER NOT NULL references shares(id),
size bigint NOT NULL,
primary key(id)
);
- });
-
- $dbh->do( qq{
create table archive (
id serial,
dvd_nr int not null,
date timestamp default now(),
primary key(id)
);
- }
- );
- $dbh->do( qq{
- create table archive_backup
- (
+ create table archive_backup (
archive_id int not null references archive(id) on delete cascade,
backup_id int not null references backups(id),
primary key(archive_id, backup_id)
);
+
+ create table archive_burned (
+ archive_id int references archive(id),
+ date timestamp default now(),
+ part int not null default 1,
+ copy int not null default 1,
+ iso_size bigint default -1
+ );
+
+ create table backup_parts (
+ id serial,
+ backup_id int references backups(id),
+ part_nr int not null check (part_nr > 0),
+ tar_size bigint not null check (tar_size > 0),
+ size bigint not null check (size > 0),
+ md5 text not null,
+ items int not null check (items > 0),
+ date timestamp default now(),
+ primary key(id)
+ );
});
print "creating indexes: ";
files:date
files:size
archive:dvd_nr
+ archive_burned:archive_id
+ backup_parts:backup_id,part_nr
)) {
do_index($index);
}
$sth->{insert_backups} = $dbh->prepare(qq{
INSERT INTO backups (hostID, num, date, type, shareid, size)
-VALUES (?,?,?,?,?,?)
+VALUES (?,?,?,?,?,-1)
+});
+
+$sth->{update_backups_size} = $dbh->prepare(qq{
+UPDATE backups SET size = ?
+WHERE hostID = ? and num = ? and date = ? and type =? and shareid = ?
});
$sth->{insert_files} = $dbh->prepare(qq{
VALUES (?,?,?,?,?,?,?)
});
-foreach my $host_key (keys %{$hosts}) {
+my @hosts = keys %{$hosts};
+my $host_nr = 0;
+
+foreach my $host_key (@hosts) {
my $hostname = $hosts->{$host_key}->{'host'} || die "can't find host for $host_key";
$hostID = $dbh->last_insert_id(undef,undef,'hosts',undef);
}
- print "host ".$hosts->{$host_key}->{'host'}.": ";
+ $host_nr++;
+ print "host ", $hosts->{$host_key}->{'host'}, " [",
+ $host_nr, "/", ($#hosts + 1), "]: ";
# get backups for a host
my @backups = $bpc->BackupInfoRead($hostname);
# dump some log
print curr_time," ", $share;
- my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);
-
$sth->{insert_backups}->execute(
$hostID,
$backupNum,
$backup->{'endTime'},
substr($backup->{'type'},0,4),
$shareID,
- $size,
);
- print " commit";
- $dbh->commit();
+ my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);
+
+ eval {
+ $sth->{update_backups_size}->execute(
+ $size,
+ $hostID,
+ $backupNum,
+ $backup->{'endTime'},
+ substr($backup->{'type'},0,4),
+ $shareID,
+ );
+ print " commit";
+ $dbh->commit();
+ };
+ if ($@) {
+ print " rollback";
+ $dbh->rollback();
+ }
my $dur = (time() - $t) || 1;
printf(" %d/%d files %d/%d dirs %0.2f MB [%.2f/s dur: %s]\n",
SELECT 1 FROM files
WHERE shareID = ? and
path = ? and
- date = ? and
- size = ?
+ size = ? and
+ ( date = ? or date = ? or date = ? )
LIMIT 1
});
- my @param = ($shareID,$path,$date,$size);
+ my @param = ($shareID,$path,$size,$date, $date-$dst_offset, $date+$dst_offset);
$sth->{file_in_db}->execute(@param);
my $rows = $sth->{file_in_db}->rows;
print STDERR "## found_in_db($shareID,$path,$date,$size) ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);
$filesInBackup->{$path_key}->{'size'}
));
+ my $key_dst_prev = join(" ", (
+ $shareID,
+ $dir,
+ $path_key,
+ $filesInBackup->{$path_key}->{'mtime'} - $dst_offset,
+ $filesInBackup->{$path_key}->{'size'}
+ ));
+
+ my $key_dst_next = join(" ", (
+ $shareID,
+ $dir,
+ $path_key,
+ $filesInBackup->{$path_key}->{'mtime'} + $dst_offset,
+ $filesInBackup->{$path_key}->{'size'}
+ ));
+
my $found;
- if (! defined($beenThere->{$key}) && ! ($found = found_in_db($key, @data)) ) {
+ if (
+ ! defined($beenThere->{$key}) &&
+ ! defined($beenThere->{$key_dst_prev}) &&
+ ! defined($beenThere->{$key_dst_next}) &&
+ ! ($found = found_in_db($key, @data))
+ ) {
print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2);
if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
projects / BackupPC.git / blobdiff