X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;f=lib%2FBackupPC%2FSearchLib.pm;h=792d5b9a6f62c63003232f7da318bf9f3829abf6;hb=bc4ccaeb2aa4b32cc135e9cf69f75f502ad50a06;hp=7f83bab38eac0fb3b95f9808c5f13a5030af7608;hpb=eb1b724fe866250a04a2f5824bf541b1730c2315;p=BackupPC.git diff --git a/lib/BackupPC/SearchLib.pm b/lib/BackupPC/SearchLib.pm index 7f83bab..792d5b9 100644 --- a/lib/BackupPC/SearchLib.pm +++ b/lib/BackupPC/SearchLib.pm @@ -8,6 +8,8 @@ use DBI; use DateTime; use vars qw(%In $MyURL); use Time::HiRes qw/time/; +use XML::Writer; +use IO::File; my $on_page = 100; my $pager_pages = 10; @@ -15,13 +17,14 @@ my $pager_pages = 10; my $dsn = $Conf{SearchDSN}; my $db_user = $Conf{SearchUser} || ''; -my $index_path = $Conf{HyperEstraierIndex}; -if ($index_path) { - $index_path = $TopDir . '/' . $index_path; - $index_path =~ s#//#/#g; -} +my $hest_node_url = $Conf{HyperEstraierIndex}; my $dbh; +my $Dir = $Conf{InstallDir}."/data/log"; +open(LOG, ">>", "$Dir/LOG") +select(LOG); +$|=1; + sub get_dbh { $dbh ||= DBI->connect($dsn, $db_user, "", { RaiseError => 1, AutoCommit => 1 } ); @@ -62,25 +65,44 @@ sub dates_from_form($) { sub mk_epoch_date($$) { my ($name,$suffix) = @_; - my $yyyy = $param->{ $name . '_year_' . $suffix} || return; + my $yyyy = $param->{ $name . '_year_' . $suffix} || return undef; my $mm .= $param->{ $name . '_month_' . $suffix} || ( $suffix eq 'from' ? 1 : 12); my $dd .= $param->{ $name . '_day_' . $suffix} || ( $suffix eq 'from' ? 1 : 31); + + $yyyy =~ s/\D//g; + $mm =~ s/\D//g; + $dd =~ s/\D//g; + + my $h = my $m = my $s = 0; + if ($suffix eq 'to') { + $h = 23; + $m = 59; + $s = 59; + } + my $dt = new DateTime( year => $yyyy, month => $mm, - day => $dd + day => $dd, + hour => $h, + minute => $m, + second => $s, ); + print STDERR "mk_epoch_date($name,$suffix) [$yyyy-$mm-$dd] = " . $dt->ymd . " " . $dt->hms . "\n"; return $dt->epoch || 'NULL'; } - return ( + my @ret = ( mk_epoch_date('search_backup', 'from'), mk_epoch_date('search_backup', 'to'), mk_epoch_date('search', 'from'), mk_epoch_date('search', 'to'), ); + + return @ret; + } @@ -95,17 +117,88 @@ sub getWhere($) { push @conditions, qq{ files.date >= $files_from } if ($files_from); push @conditions, qq{ files.date <= $files_to } if ($files_to); - print STDERR "backup: $backup_from - $backup_to files: $files_from - $files_to cond:" . join(" | ",@conditions); + print STDERR "backup: $backup_from - $backup_to files: $files_from - $files_to cond:" . join(" and ",@conditions); push( @conditions, ' files.shareid = ' . $param->{'search_share'} ) if ($param->{'search_share'}); push (@conditions, " upper(files.path) LIKE upper('%".$param->{'search_filename'}."%')") if ($param->{'search_filename'}); + if ( $param->{burned} ) { + my $is_what = 'is null'; + $is_what = '= 1' if ($param->{burned} eq 'burned'); + push @conditions, "archive_burned.part $is_what"; + push @conditions, "archive_burned.copy $is_what"; + } + return join(" and ", @conditions); } +my $sort_def = { + search => { + default => 'date_a', + sql => { + share_d => 'shares.name DESC', + share_a => 'shares.name ASC', + path_d => 'files.path DESC', + path_a => 'files.path ASC', + num_d => 'files.backupnum DESC', + num_a => 'files.backupnum ASC', + size_d => 'files.size DESC', + size_a => 'files.size ASC', + date_d => 'files.date DESC', + date_a => 'files.date ASC', + }, + est => { + share_d => 'sname STRD', + share_a => 'sname STRA', + path_d => 'filepath STRD', + path_a => 'filepath STRA', + num_d => 'backupnum NUMD', + num_a => 'backupnum NUMA', + size_d => 'size NUMD', + size_a => 'size NUMA', + date_d => 'date NUMD', + date_a => 'date NUMA', + } + }, burn => { + default => 'date_a', + sql => { + share_d => 'host DESC, share DESC', + share_a => 'host ASC, share ASC', + num_d => 'backupnum DESC', + num_a => 'backupnum ASC', + date_d => 'date DESC', + date_a => 'date ASC', + age_d => 'age DESC', + age_a => 'age ASC', + size_d => 'size DESC', + size_a => 'size ASC', + incsize_d => 'inc_size DESC', + incsize_a => 'inc_size ASC', + } + } +}; + +sub getSort($$$) { + my ($part,$type, $sort_order) = @_; + + die "unknown part: $part" unless ($sort_def->{$part}); + die "unknown type: $type" unless ($sort_def->{$part}->{$type}); + + $sort_order ||= $sort_def->{$part}->{'default'}; + + if (my $ret = $sort_def->{$part}->{$type}->{$sort_order}) { + return $ret; + } else { + # fallback to default sort order + return $sort_def->{$part}->{$type}->{ $sort_def->{$part}->{'default'} }; + } +} + +sub getFiles($) { + my ($param) = @_; -sub getFiles($$) { - my ($param, $offset) = @_; + my $offset = $param->{'offset'} || 0; + $offset *= $on_page; my $dbh = get_dbh(); @@ -113,43 +206,42 @@ sub getFiles($$) { files.id AS fid, hosts.name AS hname, shares.name AS sname, - -- shares.share AS sharename, files.backupnum AS backupnum, - -- files.name AS filename, files.path AS filepath, files.date AS date, files.type AS type, - files.size AS size, - -- dvds.name AS dvd - null AS dvd + files.size AS size }; my $sql_from = qq{ FROM files INNER JOIN shares ON files.shareID=shares.ID INNER JOIN hosts ON hosts.ID = shares.hostID - INNER JOIN backups ON backups.num = files.backupnum and backups.hostID = hosts.ID AND backups.shareID = shares.ID - }; - - my $sql_dvd_from = qq{ - -- LEFT JOIN dvds ON dvds.ID = files.dvdid + INNER JOIN backups ON backups.num = files.backupnum and backups.hostID = hosts.ID AND backups.shareID = files.shareID }; my $sql_where; my $where = getWhere($param); $sql_where = " WHERE ". $where if ($where); + # do we have to add tables for burned media? + if ( $param->{burned} ) { + $sql_from .= qq{ + LEFT OUTER JOIN archive_backup on archive_backup.backup_id = backups.id + LEFT OUTER JOIN archive_burned on archive_burned.archive_id = archive_backup.archive_id + }; + } + + my $order = getSort('search', 'sql', $param->{'sort'}); + my $sql_order = qq{ - ORDER BY files.date + ORDER BY $order LIMIT $on_page OFFSET ? }; my $sql_count = qq{ select count(files.id) $sql_from $sql_where }; - my $sql_results = qq{ select $sql_cols $sql_from $sql_dvd_from $sql_where $sql_order }; - - $offset ||= 0; - $offset = ($offset * $on_page); + my $sql_results = qq{ select $sql_cols $sql_from $sql_where $sql_order }; my $sth = $dbh->prepare($sql_count); $sth->execute(); @@ -174,61 +266,93 @@ sub getFiles($$) { return ($results, \@ret); } -sub getFilesHyperEstraier($$) { - my ($param, $offset) = @_; +sub getHyperEstraier_url($) { + my ($use_hest) = @_; - die "no index_path?" unless ($index_path); + return unless $use_hest; - use HyperEstraier; + use Search::Estraier 0.04; + die "direct access to Hyper Estraier datatase is no longer supported. Please use estmaster\n" + unless ($use_hest =~ m#^http://#); + + return $use_hest; +} + +sub getFilesHyperEstraier($) { + my ($param) = @_; + + my $offset = $param->{'offset'} || 0; + $offset *= $on_page; + + die "no Hyper Estraier node URL?" unless ($hest_node_url); # open the database - my $db = HyperEstraier::Database->new(); - $db->open($index_path, $HyperEstraier::ESTDBREADER); + my $db; + if ($hest_node_url) { + $db ||= Search::Estraier::Node->new($hest_node_url); + $db->set_auth('admin', 'admin'); + } else { + die "BUG: unimplemented"; + } # create a search condition object - my $cond = HyperEstraier::Condition->new(); + my $cond = Search::Estraier::Condition->new(); my $q = $param->{'search_filename'}; my $shareid = $param->{'search_share'}; - if ($q) { - $q =~ s/(.)/$1 /g; + if (length($q) > 0) { + # exact match + $cond->add_attr("filepath ISTRINC $q"); + $q =~ s/(.)/$1 /g; # set the search phrase to the search condition object $cond->set_phrase($q); + } - my ($backup_from, $backup_to, $files_from, $files_to) = dates_from_form($param); - - $cond->add_attr("backup_date NUMGE $backup_from") if ($backup_from); - $cond->add_attr("backup_date NUMLE $backup_to") if ($backup_to); + my ($backup_from, $backup_to, $files_from, $files_to) = dates_from_form($param); - $cond->add_attr("date NUMGE $files_from") if ($files_from); - $cond->add_attr("date NUMLE $files_to") if ($files_to); + $cond->add_attr("backup_date NUMGE $backup_from") if ($backup_from); + $cond->add_attr("backup_date NUMLE $backup_to") if ($backup_to); - $cond->add_attr("shareid NUMEQ $shareid") if ($shareid); - } + $cond->add_attr("date NUMGE $files_from") if ($files_from); + $cond->add_attr("date NUMLE $files_to") if ($files_to); - $offset ||= 0; - $offset = ($offset * $on_page); + $cond->add_attr("shareid NUMEQ $shareid") if ($shareid); -# $cond->set_max( $offset + $on_page ); - $cond->set_options( $HyperEstraier::Condition::SURE ); - $cond->set_order( 'date NUMA' ); + $cond->set_max( $offset + $on_page ); + $cond->set_options( 'SURE' ); + $cond->set_order( getSort('search', 'est', $param->{'sort'} ) ); # get the result of search - my $result = $db->search($cond, 0); - $result->get(0); - my @res; - my $hits = $result->size; + my ($result, $hits); + + if ($hest_node_url) { + $result = $db->search($cond, 0); + if ($result) { + $hits = $result->hits; + } else { + $hits = 0; + return ($hits,[]); + } + } else { + die "BUG: unimplemented"; + } # for each document in result - for my $i ($offset .. $result->size-1) { - my $id = $result->get($i); - my $doc = $db->get_doc($id, 0); + for my $i ($offset .. ($offset + $on_page - 1)) { + last if ($i >= $result->doc_num); + + my $doc; + if ($hest_node_url) { + $doc = $result->get_doc($i); + } else { + die "BUG: unimplemented"; + } my $row; - foreach my $c (qw/fid hname sname backupnum fiilename filepath date type size/) { + foreach my $c (qw/fid hname sname backupnum filepath date type size/) { $row->{$c} = $doc->attr($c); } push @res, $row; @@ -237,127 +361,586 @@ sub getFilesHyperEstraier($$) { return ($hits, \@res); } -sub getBackupsNotBurned() { +sub getGzipName($$$) +{ + my ($host, $share, $backupnum) = @_; + my $ret = $Conf{GzipSchema}; + + $share =~ s/\//_/g; + $ret =~ s/\\h/$host/ge; + $ret =~ s/\\s/$share/ge; + $ret =~ s/\\n/$backupnum/ge; + $ret =~ s/__+/_/g; + + return $ret; + +} + +sub get_tgz_size_by_name($) { + my $name = shift; + + my $tgz = $Conf{InstallDir}.'/'.$Conf{GzipTempDir}.'/'.$name; + my $size = -1; + + if (-f "${tgz}.tar.gz") { + print "stating ${tgz}.tar.gz..."; + $size = (stat("${tgz}.tar.gz"))[7]; + print "size: $size\n"; + } elsif (-d $tgz) { + print "$tgz is dir, stating files in it...\n"; + opendir(my $dir, $tgz) || die "can't opendir $tgz: $!"; + my @parts = grep { !/^\./ && !/md5/ && -f "$tgz/$_" } readdir($dir); + $size = 0; + foreach my $part (@parts) { + my $currSize = (stat("$tgz/$part"))[7]; + $size += (stat("$tgz/$part"))[7] || die "can't stat $tgz/$part: $!"; + print "\t$tgz/$part: $currSize\n"; + } + print "\ttotal $size\n"; + + closedir $dir; + } else { + return -1; + } + + return $size; +} + +sub getGzipSize($$) +{ + my ($hostID, $backupNum) = @_; + my $sql; + my $dbh = get_dbh(); + + $sql = q{ + SELECT hosts.name as host, + shares.name as share, + backups.num as backupnum + FROM hosts, backups, shares + WHERE shares.id=backups.shareid AND + hosts.id =backups.hostid AND + hosts.id=? AND + backups.num=? + }; + my $sth = $dbh->prepare($sql); + $sth->execute($hostID, $backupNum); + + my $row = $sth->fetchrow_hashref(); + + return get_tgz_size_by_name( + getGzipName($row->{'host'}, $row->{share}, $row->{'backupnum'}) + ); +} + +sub getVolumes($) { + my $id = shift; + + my $max_archive_size = $Conf{MaxArchiveSize} || die "no MaxArchiveSize"; + + my $sth = $dbh->prepare(qq{ + select + size + from backup_parts + where backup_id = ? + order by part_nr asc + }); + + $sth->execute($id); + + my $cumulative_size = 0; + my $volumes = 1; + + while(my ($size) = $sth->fetchrow_array) { + if ($cumulative_size + $size > $max_archive_size) { + $volumes++; + $cumulative_size = $size; + } else { + $cumulative_size += $size; + } + } + + return ($volumes,$cumulative_size); +} + +sub getBackupsNotBurned($) { + + my $param = shift; my $dbh = get_dbh(); - my $sql = q{ - SELECT - backups.hostID AS hostid, - min(hosts.name) AS host, - backups.num AS backupnum, - min(backups.type) AS type, - min(backups.date) AS date, - min(backups.size) AS size - FROM files - INNER JOIN shares ON files.shareID=shares.ID - INNER JOIN hosts ON hosts.ID = shares.hostID - INNER JOIN backups ON backups.num = files.backupnum and backups.hostID = hosts.ID AND backups.shareID = shares.ID - WHERE - files.dvdid IS NULL - GROUP BY - backups.hostID, backups.num - ORDER BY min(backups.date) + + my $order = getSort('burn', 'sql', $param->{'sort'}); + +print STDERR "## sort=". ($param->{'sort'} || 'no sort param') . " burn sql order: $order\n"; + + my $sql = qq{ + SELECT + backups.hostID AS hostID, + hosts.name AS host, + shares.name AS share, + backups.num AS backupnum, + backups.type AS type, + backups.date AS date, + date_part('epoch',now()) - backups.date as age, + backups.size AS size, + backups.id AS id, + backups.inc_size AS inc_size, + backups.parts AS parts + FROM backups + INNER JOIN shares ON backups.shareID=shares.ID + INNER JOIN hosts ON backups.hostID = hosts.ID + LEFT OUTER JOIN archive_backup ON archive_backup.backup_id = backups.id + WHERE backups.inc_size > 0 AND backups.size > 0 AND backups.inc_deleted is false AND archive_backup.backup_id IS NULL AND backups.parts > 0 + GROUP BY + backups.hostID, + hosts.name, + shares.name, + backups.num, + backups.shareid, + backups.id, + backups.type, + backups.date, + backups.size, + backups.inc_size, + backups.parts + ORDER BY $order }; my $sth = $dbh->prepare( $sql ); my @ret; $sth->execute(); while ( my $row = $sth->fetchrow_hashref() ) { - $row->{'age'} = sprintf("%0.1f", ( (time() - $row->{'date'}) / 86400 ) ); - $row->{'size'} = sprintf("%0.2f", $row->{'size'} / 1024 / 1024); + $row->{'age'} = sprintf("%0.1f", ( $row->{'age'} / 86400 ) ); + #$row->{'age'} = sprintf("%0.1f", ( (time() - $row->{'date'}) / 86400 ) ); + + my $max_archive_size = $Conf{MaxArchiveSize} || die "no MaxArchiveSize"; + if ($row->{size} > $max_archive_size) { + ($row->{volumes}, $row->{inc_size_calc}) = getVolumes($row->{id}); + } + + $row->{size} = sprintf("%0.2f", $row->{size} / 1024 / 1024); + + # do some cluster calculation (approximate) + $row->{inc_size} = int(( ($row->{inc_size} + 1023 ) / 2 ) * 2); + $row->{inc_size_calc} ||= $row->{inc_size}; push @ret, $row; } - return @ret; + return @ret; } -sub displayBackupsGrid() - { - my $retHTML = ""; - my $addForm = 1; - - if ($addForm) { +sub displayBackupsGrid($) { + + my $param = shift; - $retHTML .= < + my $max_archive_size = $Conf{MaxArchiveSize} || die "no MaxArchiveSize"; + my $max_archive_file_size = $Conf{MaxArchiveFileSize} || die "no MaxFileInSize"; + + my $retHTML .= q{ +
+ }; + + $retHTML .= <<'EOF3'; + + +
+ + +Size: kB + +
+
 
+
 
+
0%
+
+
+ +
 
+ +Note: + - if ($addForm) { - $retHTML .= ""; + + +
+ +EOF3 + $retHTML .= q{ + + + + + + } . + sort_header($param, 'Share', 'share', 'center') . + sort_header($param, '#', 'num', 'center') . + qq{ + + } . + sort_header($param, 'Date', 'date', 'center') . + sort_header($param, 'Age/days', 'age', 'center') . + sort_header($param, 'Size/Mb', 'size', 'center') . + sort_header($param, 'gzip size/Kb', 'incsize', 'center') . + qq{ + + }; + + my @color = (' bgcolor="#e0e0e0"', ''); + + my $i = 0; + my $host = ''; + + foreach my $backup ( getBackupsNotBurned($param) ) { + + if ($host ne $backup->{'host'}) { + $i++; + $host = $backup->{'host'}; + } + my $ftype = ""; + + my $checkbox_key = $backup->{'hostid'}. '_' .$backup->{'backupnum'} . '_' . $backup->{'id'}; + + $retHTML .= + ' + ' . + '' . + '' . + '' . + '' . + '' . + '' . + '' . + '' . + '' . + + "\n"; } + + $retHTML .= "
+ + Typemedias
'; + + if (($backup->{'inc_size'} || 0) > 0) { + $retHTML .= ' + '; + } + + my $img_url = $Conf{CgiImageDirURL}; + + $retHTML .= + '' . $backup->{'host'} . ':' . $backup->{'share'} . '' . $backup->{'backupnum'} . '' . $backup->{'type'} . '' . epoch_to_iso( $backup->{'date'} ) . '' . $backup->{'age'} . '' . $backup->{'size'} . '' . sprintf("%0.1f", $backup->{'inc_size'} / 1024 ) . + '' . ( qq{media} x $backup->{volumes} ) . '
"; + $retHTML .= ""; return $retHTML; } @@ -373,10 +956,10 @@ sub displayGrid($) { my $start_t = time(); my ($results, $files); - if ($param->{'use_hest'}) { - ($results, $files) = getFilesHyperEstraier($param, $offset); + if ($param->{'use_hest'} && length($hilite) > 0) { + ($results, $files) = getFilesHyperEstraier($param); } else { - ($results, $files) = getFiles($param, $offset); + ($results, $files) = getFiles($param); } my $dur_t = time() - $start_t; @@ -402,11 +985,54 @@ sub displayGrid($) { - - - - - + + }; + + sub sort_header($$$$) { + my ($param, $display, $name, $align) = @_; + + my ($sort_what, $sort_direction) = split(/_/,$param->{'sort'},2); + + my $old_sort = $param->{'sort'}; + + my $html = qq{'; + $param->{'sort'} = $old_sort; + + return $html; + } + + $retHTML .= + sort_header($param, 'Share', 'share', 'center') . + sort_header($param, 'Type and Name', 'path', 'center') . + sort_header($param, '#', 'num', 'center') . + sort_header($param, 'Size', 'size', 'center') . + sort_header($param, 'Date', 'date', 'center'); + + $retHTML .= qq{ }; @@ -426,17 +1052,63 @@ sub displayGrid($) { return sprintf(qq{%s}, $action, @_); } + my $sth_archived; + my %archived_cache; + + sub check_archived($$$) { + my ($host, $share, $num) = @_; + + if (my $html = $archived_cache{"$host $share $num"}) { + return $html; + } + + $sth_archived ||= $dbh->prepare(qq{ + select + dvd_nr, note, + count(archive_burned.copy) as copies + from archive + inner join archive_burned on archive_burned.archive_id = archive.id + inner join archive_backup on archive.id = archive_backup.archive_id + inner join backups on backups.id = archive_backup.backup_id + inner join hosts on hosts.id = backups.hostid + inner join shares on shares.id = backups.shareid + where hosts.name = ? and shares.name = ? and backups.num = ? + group by dvd_nr, note + }); + + my @mediums; + + $sth_archived->execute($host, $share, $num); + while (my $row = $sth_archived->fetchrow_hashref()) { + push @mediums, '' .$row->{'dvd_nr'} . + ''; + } + + my $html = join(", ",@mediums); + $archived_cache{"$host $share $num"} = $html; + return $html; + } + + my $i = $offset * $on_page; + foreach $file (@{ $files }) { + $i++; + my $typeStr = BackupPC::Attrib::fileType2Text(undef, $file->{'type'}); $retHTML .= qq{}; + $retHTML .= qq{}; + $retHTML .= qq{} . qq{} . qq{} . qq{} . qq{} . - qq{}; + qq{}; $retHTML .= ""; } @@ -451,20 +1123,26 @@ sub displayGrid($) { my $max_page = int( $results / $on_page ); my $page = 0; - sub page_link($$$) { - my ($param,$page,$display) = @_; - - $param->{'offset'} = $page; + sub page_uri($) { + my $param = shift || die "no param?"; - my $html = '' . $display . ''; + return $uri; + } + + sub page_link($$$) { + my ($param,$page,$display) = @_; + + $param->{'offset'} = $page if (defined($page)); + + my $html = '' . $display . ''; } $retHTML .= '
';
ShareType and Name#SizeDate{'sort'} = $name . '_' . $direction; + $html .= ' style="border: 1px solid #808080;"'; + + # add unicode arrow for direction + $arrow .= ' '; + $arrow .= $direction eq 'a' ? '▲' + : $direction eq 'd' ? '▼' + : '' + ; + + } else { + $param->{'sort'} = $name . '_a'; + } + + $html .= '>' . $display . '' . $arrow . 'Media
$i} . $file->{'hname'} . ':' . $file->{'sname'} . qq{$typeStr } . hilite_html( $file->{'filepath'}, $hilite ) . qq{} . restore_link( $typeStr, ${EscURI( $file->{'hname'} )}, $file->{'backupnum'}, ${EscURI( $file->{'sname'})}, ${EscURI( $file->{'filepath'} )}, $file->{'backupnum'} ) . qq{} . $file->{'size'} . qq{} . epoch_to_iso( $file->{'date'} ) . qq{} . $file->{'dvd'} . qq{} . check_archived( $file->{'hname'}, $file->{'sname'}, $file->{'backupnum'} ) . qq{