X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;f=lib%2FBackupPC%2FSearchLib.pm;h=5044b4f4536bd4abc4d80233cba63a8a71a2b68e;hb=41c3e69d275ebd963b549df97dcbeeeea6f7dec2;hp=6f88aa03d7714a1dff126132fea6d97f0a7ad4f1;hpb=973aeb910bfec685dba46563e6613f38d8a3ce34;p=BackupPC.git diff --git a/lib/BackupPC/SearchLib.pm b/lib/BackupPC/SearchLib.pm index 6f88aa0..5044b4f 100644 --- a/lib/BackupPC/SearchLib.pm +++ b/lib/BackupPC/SearchLib.pm @@ -8,6 +8,8 @@ use DBI; use DateTime; use vars qw(%In $MyURL); use Time::HiRes qw/time/; +use XML::Writer; +use IO::File; my $on_page = 100; my $pager_pages = 10; @@ -15,7 +17,7 @@ my $pager_pages = 10; my $dsn = $Conf{SearchDSN}; my $db_user = $Conf{SearchUser} || ''; -my $hest_index_path = $Conf{HyperEstraierIndex}; +my $hest_node_url = $Conf{HyperEstraierIndex}; my $dbh; @@ -68,10 +70,20 @@ sub dates_from_form($) { $mm =~ s/\D//g; $dd =~ s/\D//g; + my $h = my $m = my $s = 0; + if ($suffix eq 'to') { + $h = 23; + $m = 59; + $s = 59; + } + my $dt = new DateTime( year => $yyyy, month => $mm, - day => $dd + day => $dd, + hour => $h, + minute => $m, + second => $s, ); print STDERR "mk_epoch_date($name,$suffix) [$yyyy-$mm-$dd] = " . $dt->ymd . " " . $dt->hms . "\n"; return $dt->epoch || 'NULL'; @@ -100,7 +112,7 @@ sub getWhere($) { push @conditions, qq{ files.date >= $files_from } if ($files_from); push @conditions, qq{ files.date <= $files_to } if ($files_to); - print STDERR "backup: $backup_from - $backup_to files: $files_from - $files_to cond:" . join(" | ",@conditions); + print STDERR "backup: $backup_from - $backup_to files: $files_from - $files_to cond:" . join(" and ",@conditions); push( @conditions, ' files.shareid = ' . $param->{'search_share'} ) if ($param->{'search_share'}); push (@conditions, " upper(files.path) LIKE upper('%".$param->{'search_filename'}."%')") if ($param->{'search_filename'}); @@ -108,6 +120,67 @@ sub getWhere($) { return join(" and ", @conditions); } +my $sort_def = { + search => { + default => 'date_a', + sql => { + share_d => 'shares.name DESC', + share_a => 'shares.name ASC', + path_d => 'files.path DESC', + path_a => 'files.path ASC', + num_d => 'files.backupnum DESC', + num_a => 'files.backupnum ASC', + size_d => 'files.size DESC', + size_a => 'files.size ASC', + date_d => 'files.date DESC', + date_a => 'files.date ASC', + }, + est => { + share_d => 'sname STRD', + share_a => 'sname STRA', + path_d => 'filepath STRD', + path_a => 'filepath STRA', + num_d => 'backupnum NUMD', + num_a => 'backupnum NUMA', + size_d => 'size NUMD', + size_a => 'size NUMA', + date_d => 'date NUMD', + date_a => 'date NUMA', + } + }, burn => { + default => 'date_a', + sql => { + share_d => 'host DESC, share DESC', + share_a => 'host ASC, share ASC', + num_d => 'backupnum DESC', + num_a => 'backupnum ASC', + date_d => 'date DESC', + date_a => 'date ASC', + age_d => 'age DESC', + age_a => 'age ASC', + size_d => 'size DESC', + size_a => 'size ASC', + incsize_d => 'inc_size DESC', + incsize_a => 'inc_size ASC', + } + } +}; + +sub getSort($$$) { + my ($part,$type, $sort_order) = @_; + + die "unknown part: $part" unless ($sort_def->{$part}); + die "unknown type: $type" unless ($sort_def->{$part}->{$type}); + + $sort_order ||= $sort_def->{$part}->{'default'}; + + if (my $ret = $sort_def->{$part}->{$type}->{$sort_order}) { + return $ret; + } else { + # fallback to default sort order + return $sort_def->{$part}->{$type}->{ $sort_def->{$part}->{'default'} }; + } +} sub getFiles($) { my ($param) = @_; @@ -139,8 +212,10 @@ sub getFiles($) { my $where = getWhere($param); $sql_where = " WHERE ". $where if ($where); + my $order = getSort('search', 'sql', $param->{'sort'}); + my $sql_order = qq{ - ORDER BY files.date + ORDER BY $order LIMIT $on_page OFFSET ? }; @@ -176,16 +251,11 @@ sub getHyperEstraier_url($) { return unless $use_hest; - use HyperEstraier; - my ($index_path, $index_node_url); + use Search::Estraier 0.04; + die "direct access to Hyper Estraier datatase is no longer supported. Please use estmaster\n" + unless ($use_hest =~ m#^http://#); - if ($use_hest =~ m#^http://#) { - $index_node_url = $use_hest; - } else { - $index_path = $TopDir . '/' . $index_path; - $index_path =~ s#//#/#g; - } - return ($index_path, $index_node_url); + return $use_hest; } sub getFilesHyperEstraier($) { @@ -194,26 +264,19 @@ sub getFilesHyperEstraier($) { my $offset = $param->{'offset'} || 0; $offset *= $on_page; - die "no index_path?" unless ($hest_index_path); - - use HyperEstraier; - - my ($index_path, $index_node_url) = getHyperEstraier_url($hest_index_path); + die "no Hyper Estraier node URL?" unless ($hest_node_url); # open the database my $db; - if ($index_path) { - $db = HyperEstraier::Database->new(); - $db->open($index_path, $HyperEstraier::ESTDBREADER); - } elsif ($index_node_url) { - $db ||= HyperEstraier::Node->new($index_node_url); + if ($hest_node_url) { + $db ||= Search::Estraier::Node->new($hest_node_url); $db->set_auth('admin', 'admin'); } else { die "BUG: unimplemented"; } # create a search condition object - my $cond = HyperEstraier::Condition->new(); + my $cond = Search::Estraier::Condition->new(); my $q = $param->{'search_filename'}; my $shareid = $param->{'search_share'}; @@ -237,40 +300,39 @@ sub getFilesHyperEstraier($) { $cond->add_attr("shareid NUMEQ $shareid") if ($shareid); -# $cond->set_max( $offset + $on_page ); - $cond->set_options( $HyperEstraier::Condition::SURE ); - $cond->set_order( 'date NUMA' ); + $cond->set_max( $offset + $on_page ); + $cond->set_options( 'SURE' ); + $cond->set_order( getSort('search', 'est', $param->{'sort'} ) ); # get the result of search my @res; my ($result, $hits); - if ($index_path) { + if ($hest_node_url) { $result = $db->search($cond, 0); - $hits = $result->size; - } elsif ($index_node_url) { - $result = $db->search($cond, 0); - $hits = $result->doc_num; + if ($result) { + $hits = $result->hits; + } else { + $hits = 0; + return ($hits,[]); + } } else { die "BUG: unimplemented"; } # for each document in result for my $i ($offset .. ($offset + $on_page - 1)) { - last if ($i >= $hits); + last if ($i >= $result->doc_num); my $doc; - if ($index_path) { - my $id = $result->get($i); - $doc = $db->get_doc($id, 0); - } elsif ($index_node_url) { + if ($hest_node_url) { $doc = $result->get_doc($i); } else { die "BUG: unimplemented"; } my $row; - foreach my $c (qw/fid hname sname backupnum fiilename filepath date type size/) { + foreach my $c (qw/fid hname sname backupnum filepath date type size/) { $row->{$c} = $doc->attr($c); } push @res, $row; @@ -295,10 +357,33 @@ sub getGzipName($$$) } +sub get_tgz_size_by_name($) { + my $name = shift; + + my $tgz = $Conf{InstallDir}.'/'.$Conf{GzipTempDir}.'/'.$name; + + my $size = -1; + + if (-f "${tgz}.tar.gz") { + $size = (stat("${tgz}.tar.gz"))[7]; + } elsif (-d $tgz) { + opendir(my $dir, $tgz) || die "can't opendir $tgz: $!"; + my @parts = grep { !/^\./ && !/md5/ && -f "$tgz/$_" } readdir($dir); + $size = 0; + foreach my $part (@parts) { + $size += (stat("$tgz/$part"))[7] || die "can't stat $tgz/$part: $!"; + } + closedir $dir; + } else { + return -1; + } + + return $size; +} + sub getGzipSize($$) { my ($hostID, $backupNum) = @_; - my $ret; my $sql; my $dbh = get_dbh(); @@ -316,19 +401,52 @@ sub getGzipSize($$) $sth->execute($hostID, $backupNum); my $row = $sth->fetchrow_hashref(); - - my (undef,undef,undef,undef,undef,undef,undef,$ret,undef,undef,undef,undef,undef) = - stat( $Conf{InstallDir}.'/'.$Conf{GzipTempDir}.'/'. - getGzipName($row->{'host'}, $row->{share}, $row->{'backupnum'})); - - return $ret; + + return get_tgz_size_by_name( + getGzipName($row->{'host'}, $row->{share}, $row->{'backupnum'}) + ); +} + +sub getVolumes($) { + my $id = shift; + + my $max_archive_size = $Conf{MaxArchiveSize} || die "no MaxArchiveSize"; + + my $sth = $dbh->prepare(qq{ + select + size + from backup_parts + where backup_id = ? + order by part_nr asc + }); + + $sth->execute($id); + + my $cumulative_size = 0; + my $volumes = 1; + + while(my ($size) = $sth->fetchrow_array) { + if ($cumulative_size + $size > $max_archive_size) { + $volumes++; + $cumulative_size = $size; + } else { + $cumulative_size += $size; + } + } + + return ($volumes,$cumulative_size); } -sub getBackupsNotBurned() { +sub getBackupsNotBurned($) { + my $param = shift; my $dbh = get_dbh(); - my $sql = q{ + my $order = getSort('burn', 'sql', $param->{'sort'}); + +print STDERR "## sort=". ($param->{'sort'} || 'no sort param') . " burn sql order: $order\n"; + + my $sql = qq{ SELECT backups.hostID AS hostID, hosts.name AS host, @@ -336,14 +454,16 @@ sub getBackupsNotBurned() { backups.num AS backupnum, backups.type AS type, backups.date AS date, + date_part('epoch',now()) - backups.date as age, backups.size AS size, backups.id AS id, - backups.inc_size AS inc_size + backups.inc_size AS inc_size, + backups.parts AS parts FROM backups INNER JOIN shares ON backups.shareID=shares.ID INNER JOIN hosts ON backups.hostID = hosts.ID LEFT OUTER JOIN archive_backup ON archive_backup.backup_id = backups.id - WHERE backups.size > 0 AND backups.inc_size > 0 AND archive_backup.backup_id IS NULL + WHERE backups.inc_size > 0 AND backups.size > 0 AND backups.inc_deleted is false AND archive_backup.backup_id IS NULL AND backups.parts > 0 GROUP BY backups.hostID, hosts.name, @@ -354,26 +474,40 @@ sub getBackupsNotBurned() { backups.type, backups.date, backups.size, - backups.inc_size - ORDER BY backups.date + backups.inc_size, + backups.parts + ORDER BY $order }; my $sth = $dbh->prepare( $sql ); my @ret; $sth->execute(); while ( my $row = $sth->fetchrow_hashref() ) { - $row->{'age'} = sprintf("%0.1f", ( (time() - $row->{'date'}) / 86400 ) ); - $row->{'size'} = sprintf("%0.2f", $row->{'size'} / 1024 / 1024); + $row->{'age'} = sprintf("%0.1f", ( $row->{'age'} / 86400 ) ); + #$row->{'age'} = sprintf("%0.1f", ( (time() - $row->{'date'}) / 86400 ) ); - # do some cluster calculation (approximate) and convert to kB - $row->{'inc_size'} = int($row->{'inc_size'} / ( 2 * 1024) * 2); + my $max_archive_size = $Conf{MaxArchiveSize} || die "no MaxArchiveSize"; + if ($row->{size} > $max_archive_size) { + ($row->{volumes}, $row->{inc_size_calc}) = getVolumes($row->{id}); + } + + $row->{size} = sprintf("%0.2f", $row->{size} / 1024 / 1024); + + # do some cluster calculation (approximate) + $row->{inc_size} = int(( ($row->{inc_size} + 1023 ) / 2 ) * 2); + $row->{inc_size_calc} ||= $row->{inc_size}; push @ret, $row; } - return @ret; + return @ret; } -sub displayBackupsGrid() { +sub displayBackupsGrid($) { + + my $param = shift; + + my $max_archive_size = $Conf{MaxArchiveSize} || die "no MaxArchiveSize"; + my $max_archive_file_size = $Conf{MaxArchiveFileSize} || die "no MaxFileInSize"; my $retHTML .= q{
@@ -480,6 +614,15 @@ DIV#fixedBox #submitBurner { z-index: 3; background-color: transparent; } + +#volumes { + padding: 0.4em; + display: none; + width: 100%; + font-size: 80%; + color: #ff0000; + text-align: center; +} -->
-Size: kB + +Size: kB
 
@@ -663,6 +835,8 @@ Size:  
+ Note: @@ -683,15 +857,18 @@ EOF3 - Share - Backup no + } . + sort_header($param, 'Share', 'share', 'center') . + sort_header($param, '#', 'num', 'center') . + qq{ Type - date - age/days - size/MB - gzip size/kB - - + } . + sort_header($param, 'Date', 'date', 'center') . + sort_header($param, 'Age/days', 'age', 'center') . + sort_header($param, 'Size/Mb', 'size', 'center') . + sort_header($param, 'gzip size/Kb', 'incsize', 'center') . + qq{ + medias }; my @color = (' bgcolor="#e0e0e0"', ''); @@ -699,7 +876,7 @@ EOF3 my $i = 0; my $host = ''; - foreach my $backup ( getBackupsNotBurned() ) { + foreach my $backup ( getBackupsNotBurned($param) ) { if ($host ne $backup->{'host'}) { $i++; @@ -718,6 +895,8 @@ EOF3 '; } + my $img_url = $Conf{CgiImageDirURL}; + $retHTML .= '' . '' . $backup->{'host'} . ':' . $backup->{'share'} . '' . @@ -726,8 +905,10 @@ EOF3 '' . epoch_to_iso( $backup->{'date'} ) . '' . '' . $backup->{'age'} . '' . '' . $backup->{'size'} . '' . - '' . $backup->{'inc_size'} . - '' . + '' . sprintf("%0.1f", $backup->{'inc_size'} / 1024 ) . + '' . + '' . + '' . ( qq{media} x $backup->{volumes} ) . '' . "\n"; } @@ -779,11 +960,53 @@ sub displayGrid($) { - - - - - + }; + + sub sort_header($$$$) { + my ($param, $display, $name, $align) = @_; + + my ($sort_what, $sort_direction) = split(/_/,$param->{'sort'},2); + + my $old_sort = $param->{'sort'}; + + my $html = qq{'; + $param->{'sort'} = $old_sort; + + return $html; + } + + $retHTML .= + sort_header($param, 'Share', 'share', 'center') . + sort_header($param, 'Type and Name', 'path', 'center') . + sort_header($param, '#', 'num', 'center') . + sort_header($param, 'Size', 'size', 'center') . + sort_header($param, 'Date', 'date', 'center'); + + $retHTML .= qq{ }; @@ -803,6 +1026,46 @@ sub displayGrid($) { return sprintf(qq{%s}, $action, @_); } + my $sth_archived; + my %archived_cache; + + sub check_archived($$$) { + my ($host, $share, $num) = @_; + + if (my $html = $archived_cache{"$host $share $num"}) { + return $html; + } + + $sth_archived ||= $dbh->prepare(qq{ + select + dvd_nr, note, + count(archive_burned.copy) as copies + from archive + inner join archive_burned on archive_burned.archive_id = archive.id + inner join archive_backup on archive.id = archive_backup.archive_id + inner join backups on backups.id = archive_backup.backup_id + inner join hosts on hosts.id = backups.hostid + inner join shares on shares.id = backups.shareid + where hosts.name = ? and shares.name = ? and backups.num = ? + group by dvd_nr, note + }); + + my @mediums; + + $sth_archived->execute($host, $share, $num); + while (my $row = $sth_archived->fetchrow_hashref()) { + push @mediums, '' .$row->{'dvd_nr'} . + ''; + } + + my $html = join(", ",@mediums); + $archived_cache{"$host $share $num"} = $html; + return $html; + } + my $i = $offset * $on_page; foreach $file (@{ $files }) { @@ -819,7 +1082,7 @@ sub displayGrid($) { qq{} . qq{} . qq{} . - qq{}; + qq{}; $retHTML .= ""; } @@ -834,20 +1097,26 @@ sub displayGrid($) { my $max_page = int( $results / $on_page ); my $page = 0; - sub page_link($$$) { - my ($param,$page,$display) = @_; - - $param->{'offset'} = $page; + sub page_uri($) { + my $param = shift || die "no param?"; - my $html = '' . $display . ''; + return $uri; + } + + sub page_link($$$) { + my ($param,$page,$display) = @_; + + $param->{'offset'} = $page if (defined($page)); + + my $html = '' . $display . ''; } $retHTML .= '
';
ShareType and Name#SizeDate{'sort'} = $name . '_' . $direction; + $html .= ' style="border: 1px solid #808080;"'; + + # add unicode arrow for direction + $arrow .= ' '; + $arrow .= $direction eq 'a' ? '▲' + : $direction eq 'd' ? '▼' + : '' + ; + + } else { + $param->{'sort'} = $name . '_a'; + } + + $html .= '>' . $display . '' . $arrow . 'Media
} . restore_link( $typeStr, ${EscURI( $file->{'hname'} )}, $file->{'backupnum'}, ${EscURI( $file->{'sname'})}, ${EscURI( $file->{'filepath'} )}, $file->{'backupnum'} ) . qq{} . $file->{'size'} . qq{} . epoch_to_iso( $file->{'date'} ) . qq{} . '?' . qq{} . check_archived( $file->{'hname'}, $file->{'sname'}, $file->{'backupnum'} ) . qq{