X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;f=lib%2FBackupPC%2FSearchLib.pm;h=792d5b9a6f62c63003232f7da318bf9f3829abf6;hb=bc4ccaeb2aa4b32cc135e9cf69f75f502ad50a06;hp=82b3886b3a9b89c2456ac96e537cf6e78164a9d7;hpb=51a9c2d3e2daae63a87be4f2c6e199b7e158f83f;p=BackupPC.git diff --git a/lib/BackupPC/SearchLib.pm b/lib/BackupPC/SearchLib.pm index 82b3886..792d5b9 100644 --- a/lib/BackupPC/SearchLib.pm +++ b/lib/BackupPC/SearchLib.pm @@ -8,6 +8,8 @@ use DBI; use DateTime; use vars qw(%In $MyURL); use Time::HiRes qw/time/; +use XML::Writer; +use IO::File; my $on_page = 100; my $pager_pages = 10; @@ -15,9 +17,14 @@ my $pager_pages = 10; my $dsn = $Conf{SearchDSN}; my $db_user = $Conf{SearchUser} || ''; -my $hest_index_path = $Conf{HyperEstraierIndex}; +my $hest_node_url = $Conf{HyperEstraierIndex}; my $dbh; +my $Dir = $Conf{InstallDir}."/data/log"; +open(LOG, ">>", "$Dir/LOG") +select(LOG); +$|=1; + sub get_dbh { $dbh ||= DBI->connect($dsn, $db_user, "", { RaiseError => 1, AutoCommit => 1 } ); @@ -68,10 +75,20 @@ sub dates_from_form($) { $mm =~ s/\D//g; $dd =~ s/\D//g; + my $h = my $m = my $s = 0; + if ($suffix eq 'to') { + $h = 23; + $m = 59; + $s = 59; + } + my $dt = new DateTime( year => $yyyy, month => $mm, - day => $dd + day => $dd, + hour => $h, + minute => $m, + second => $s, ); print STDERR "mk_epoch_date($name,$suffix) [$yyyy-$mm-$dd] = " . $dt->ymd . " " . $dt->hms . "\n"; return $dt->epoch || 'NULL'; @@ -100,14 +117,82 @@ sub getWhere($) { push @conditions, qq{ files.date >= $files_from } if ($files_from); push @conditions, qq{ files.date <= $files_to } if ($files_to); - print STDERR "backup: $backup_from - $backup_to files: $files_from - $files_to cond:" . join(" | ",@conditions); + print STDERR "backup: $backup_from - $backup_to files: $files_from - $files_to cond:" . join(" and ",@conditions); push( @conditions, ' files.shareid = ' . $param->{'search_share'} ) if ($param->{'search_share'}); push (@conditions, " upper(files.path) LIKE upper('%".$param->{'search_filename'}."%')") if ($param->{'search_filename'}); + if ( $param->{burned} ) { + my $is_what = 'is null'; + $is_what = '= 1' if ($param->{burned} eq 'burned'); + push @conditions, "archive_burned.part $is_what"; + push @conditions, "archive_burned.copy $is_what"; + } + return join(" and ", @conditions); } +my $sort_def = { + search => { + default => 'date_a', + sql => { + share_d => 'shares.name DESC', + share_a => 'shares.name ASC', + path_d => 'files.path DESC', + path_a => 'files.path ASC', + num_d => 'files.backupnum DESC', + num_a => 'files.backupnum ASC', + size_d => 'files.size DESC', + size_a => 'files.size ASC', + date_d => 'files.date DESC', + date_a => 'files.date ASC', + }, + est => { + share_d => 'sname STRD', + share_a => 'sname STRA', + path_d => 'filepath STRD', + path_a => 'filepath STRA', + num_d => 'backupnum NUMD', + num_a => 'backupnum NUMA', + size_d => 'size NUMD', + size_a => 'size NUMA', + date_d => 'date NUMD', + date_a => 'date NUMA', + } + }, burn => { + default => 'date_a', + sql => { + share_d => 'host DESC, share DESC', + share_a => 'host ASC, share ASC', + num_d => 'backupnum DESC', + num_a => 'backupnum ASC', + date_d => 'date DESC', + date_a => 'date ASC', + age_d => 'age DESC', + age_a => 'age ASC', + size_d => 'size DESC', + size_a => 'size ASC', + incsize_d => 'inc_size DESC', + incsize_a => 'inc_size ASC', + } + } +}; + +sub getSort($$$) { + my ($part,$type, $sort_order) = @_; + + die "unknown part: $part" unless ($sort_def->{$part}); + die "unknown type: $type" unless ($sort_def->{$part}->{$type}); + + $sort_order ||= $sort_def->{$part}->{'default'}; + + if (my $ret = $sort_def->{$part}->{$type}->{$sort_order}) { + return $ret; + } else { + # fallback to default sort order + return $sort_def->{$part}->{$type}->{ $sort_def->{$part}->{'default'} }; + } +} sub getFiles($) { my ($param) = @_; @@ -139,8 +224,18 @@ sub getFiles($) { my $where = getWhere($param); $sql_where = " WHERE ". $where if ($where); + # do we have to add tables for burned media? + if ( $param->{burned} ) { + $sql_from .= qq{ + LEFT OUTER JOIN archive_backup on archive_backup.backup_id = backups.id + LEFT OUTER JOIN archive_burned on archive_burned.archive_id = archive_backup.archive_id + }; + } + + my $order = getSort('search', 'sql', $param->{'sort'}); + my $sql_order = qq{ - ORDER BY files.date + ORDER BY $order LIMIT $on_page OFFSET ? }; @@ -176,16 +271,11 @@ sub getHyperEstraier_url($) { return unless $use_hest; - use HyperEstraier; - my ($index_path, $index_node_url); + use Search::Estraier 0.04; + die "direct access to Hyper Estraier datatase is no longer supported. Please use estmaster\n" + unless ($use_hest =~ m#^http://#); - if ($use_hest =~ m#^http://#) { - $index_node_url = $use_hest; - } else { - $index_path = $TopDir . '/' . $index_path; - $index_path =~ s#//#/#g; - } - return ($index_path, $index_node_url); + return $use_hest; } sub getFilesHyperEstraier($) { @@ -194,26 +284,19 @@ sub getFilesHyperEstraier($) { my $offset = $param->{'offset'} || 0; $offset *= $on_page; - die "no index_path?" unless ($hest_index_path); - - use HyperEstraier; - - my ($index_path, $index_node_url) = getHyperEstraier_url($hest_index_path); + die "no Hyper Estraier node URL?" unless ($hest_node_url); # open the database my $db; - if ($index_path) { - $db = HyperEstraier::Database->new(); - $db->open($index_path, $HyperEstraier::ESTDBREADER); - } elsif ($index_node_url) { - $db ||= HyperEstraier::Node->new($index_node_url); + if ($hest_node_url) { + $db ||= Search::Estraier::Node->new($hest_node_url); $db->set_auth('admin', 'admin'); } else { die "BUG: unimplemented"; } # create a search condition object - my $cond = HyperEstraier::Condition->new(); + my $cond = Search::Estraier::Condition->new(); my $q = $param->{'search_filename'}; my $shareid = $param->{'search_share'}; @@ -237,40 +320,39 @@ sub getFilesHyperEstraier($) { $cond->add_attr("shareid NUMEQ $shareid") if ($shareid); -# $cond->set_max( $offset + $on_page ); - $cond->set_options( $HyperEstraier::Condition::SURE ); - $cond->set_order( 'date NUMA' ); + $cond->set_max( $offset + $on_page ); + $cond->set_options( 'SURE' ); + $cond->set_order( getSort('search', 'est', $param->{'sort'} ) ); # get the result of search my @res; my ($result, $hits); - if ($index_path) { + if ($hest_node_url) { $result = $db->search($cond, 0); - $hits = $result->size; - } elsif ($index_node_url) { - $result = $db->search($cond, 0); - $hits = $result->doc_num; + if ($result) { + $hits = $result->hits; + } else { + $hits = 0; + return ($hits,[]); + } } else { die "BUG: unimplemented"; } # for each document in result for my $i ($offset .. ($offset + $on_page - 1)) { - last if ($i >= $hits); + last if ($i >= $result->doc_num); my $doc; - if ($index_path) { - my $id = $result->get($i); - $doc = $db->get_doc($id, 0); - } elsif ($index_node_url) { + if ($hest_node_url) { $doc = $result->get_doc($i); } else { die "BUG: unimplemented"; } my $row; - foreach my $c (qw/fid hname sname backupnum fiilename filepath date type size/) { + foreach my $c (qw/fid hname sname backupnum filepath date type size/) { $row->{$c} = $doc->attr($c); } push @res, $row; @@ -288,29 +370,126 @@ sub getGzipName($$$) $ret =~ s/\\h/$host/ge; $ret =~ s/\\s/$share/ge; $ret =~ s/\\n/$backupnum/ge; - + + $ret =~ s/__+/_/g; + return $ret; } -sub getBackupsNotBurned() { +sub get_tgz_size_by_name($) { + my $name = shift; + + my $tgz = $Conf{InstallDir}.'/'.$Conf{GzipTempDir}.'/'.$name; + my $size = -1; + + if (-f "${tgz}.tar.gz") { + print "stating ${tgz}.tar.gz..."; + $size = (stat("${tgz}.tar.gz"))[7]; + print "size: $size\n"; + } elsif (-d $tgz) { + print "$tgz is dir, stating files in it...\n"; + opendir(my $dir, $tgz) || die "can't opendir $tgz: $!"; + my @parts = grep { !/^\./ && !/md5/ && -f "$tgz/$_" } readdir($dir); + $size = 0; + foreach my $part (@parts) { + my $currSize = (stat("$tgz/$part"))[7]; + $size += (stat("$tgz/$part"))[7] || die "can't stat $tgz/$part: $!"; + print "\t$tgz/$part: $currSize\n"; + } + print "\ttotal $size\n"; + + closedir $dir; + } else { + return -1; + } + + return $size; +} + +sub getGzipSize($$) +{ + my ($hostID, $backupNum) = @_; + my $sql; + my $dbh = get_dbh(); + + $sql = q{ + SELECT hosts.name as host, + shares.name as share, + backups.num as backupnum + FROM hosts, backups, shares + WHERE shares.id=backups.shareid AND + hosts.id =backups.hostid AND + hosts.id=? AND + backups.num=? + }; + my $sth = $dbh->prepare($sql); + $sth->execute($hostID, $backupNum); + + my $row = $sth->fetchrow_hashref(); + + return get_tgz_size_by_name( + getGzipName($row->{'host'}, $row->{share}, $row->{'backupnum'}) + ); +} + +sub getVolumes($) { + my $id = shift; + + my $max_archive_size = $Conf{MaxArchiveSize} || die "no MaxArchiveSize"; + my $sth = $dbh->prepare(qq{ + select + size + from backup_parts + where backup_id = ? + order by part_nr asc + }); + + $sth->execute($id); + + my $cumulative_size = 0; + my $volumes = 1; + + while(my ($size) = $sth->fetchrow_array) { + if ($cumulative_size + $size > $max_archive_size) { + $volumes++; + $cumulative_size = $size; + } else { + $cumulative_size += $size; + } + } + + return ($volumes,$cumulative_size); +} + +sub getBackupsNotBurned($) { + + my $param = shift; my $dbh = get_dbh(); - my $sql = q{ + my $order = getSort('burn', 'sql', $param->{'sort'}); + +print STDERR "## sort=". ($param->{'sort'} || 'no sort param') . " burn sql order: $order\n"; + + my $sql = qq{ SELECT backups.hostID AS hostID, hosts.name AS host, shares.name AS share, - backups.id AS backupnum, + backups.num AS backupnum, backups.type AS type, backups.date AS date, - backups.size AS size + date_part('epoch',now()) - backups.date as age, + backups.size AS size, + backups.id AS id, + backups.inc_size AS inc_size, + backups.parts AS parts FROM backups INNER JOIN shares ON backups.shareID=shares.ID INNER JOIN hosts ON backups.hostID = hosts.ID - LEFT OUTER JOIN archive_backup ON archive_backup.backup_id = backups.id AND archive_backup.backup_id IS NULL - WHERE backups.size > 0 + LEFT OUTER JOIN archive_backup ON archive_backup.backup_id = backups.id + WHERE backups.inc_size > 0 AND backups.size > 0 AND backups.inc_deleted is false AND archive_backup.backup_id IS NULL AND backups.parts > 0 GROUP BY backups.hostID, hosts.name, @@ -320,94 +499,402 @@ sub getBackupsNotBurned() { backups.id, backups.type, backups.date, - backups.size - ORDER BY backups.date + backups.size, + backups.inc_size, + backups.parts + ORDER BY $order }; my $sth = $dbh->prepare( $sql ); my @ret; $sth->execute(); while ( my $row = $sth->fetchrow_hashref() ) { - $row->{'age'} = sprintf("%0.1f", ( (time() - $row->{'date'}) / 86400 ) ); - $row->{'size'} = sprintf("%0.2f", $row->{'size'} / 1024 / 1024); - my (undef,undef,undef,undef,undef,undef,undef,$fs_size,undef,undef,undef,undef,undef) = - stat( $Conf{InstallDir}.'/'.$Conf{GzipTempDir}.'/'. - getGzipName($row->{'host'}, $row->{share}, $row->{'backupnum'})); - $row->{'fs_size'} = $fs_size; + $row->{'age'} = sprintf("%0.1f", ( $row->{'age'} / 86400 ) ); + #$row->{'age'} = sprintf("%0.1f", ( (time() - $row->{'date'}) / 86400 ) ); + + my $max_archive_size = $Conf{MaxArchiveSize} || die "no MaxArchiveSize"; + if ($row->{size} > $max_archive_size) { + ($row->{volumes}, $row->{inc_size_calc}) = getVolumes($row->{id}); + } + + $row->{size} = sprintf("%0.2f", $row->{size} / 1024 / 1024); + + # do some cluster calculation (approximate) + $row->{inc_size} = int(( ($row->{inc_size} + 1023 ) / 2 ) * 2); + $row->{inc_size_calc} ||= $row->{inc_size}; push @ret, $row; } - return @ret; + return @ret; } -sub displayBackupsGrid() - { - my $retHTML = ""; - - $retHTML .= < +sub displayBackupsGrid($) { + + my $param = shift; + + my $max_archive_size = $Conf{MaxArchiveSize} || die "no MaxArchiveSize"; + my $max_archive_file_size = $Conf{MaxArchiveFileSize} || die "no MaxFileInSize"; + + my $retHTML .= q{ +
+ }; + + $retHTML .= <<'EOF3'; + + +} + +/* progress bar */ + +var _pbar_width = null; +var _pbar_warn = 10; // change color in last 10% + +function pbar_reset() { + element_id("mask").style.left = "0px"; + _pbar_width = element_id("mContainer").offsetWidth - 2; + element_id("mask").style.width = _pbar_width + "px"; + element_id("mask").style.display = "block"; + element_id("progressIndicator").style.zIndex = 10; + element_id("progressIndicator").innerHTML = "0"; +} + +function dec2hex(d) { + var hch = '0123456789ABCDEF'; + var a = d % 16; + var q = (d - a) / 16; + return hch.charAt(q) + hch.charAt(a); +} + +function pbar_set(amount, max) { + debug('pbar_set('+amount+', '+max+')'); + + if (_pbar_width == null) { + var _mc = element_id("mContainer"); + if (_pbar_width == null) _pbar_width = parseInt(_mc.offsetWidth ? (_mc.offsetWidth - 2) : 0) || null; + if (_pbar_width == null) _pbar_width = parseInt(_mc.clientWidth ? (_mc.clientWidth + 2) : 0) || null; + if (_pbar_width == null) _pbar_width = 0; + } + + var pcnt = Math.floor(amount * 100 / max); + var p90 = 100 - _pbar_warn; + var pcol = pcnt - p90; + if (Math.round(pcnt) <= 100) { + if (pcol < 0) pcol = 0; + var e = element_id("submitBurner"); + debug('enable_button'); + e.disabled = false; + var a = e.getAttributeNode('disabled') || null; + if (a) e.removeAttributeNode(a); + } else { + debug('disable button'); + pcol = _pbar_warn; + var e = element_id("submitBurner"); + if (!e.disabled) e.disabled = true; + } + var col_g = Math.floor((_pbar_warn - pcol) * 255 / _pbar_warn); + var col = '#FF' + dec2hex(col_g) + '00'; + + //debug('pcol: '+pcol+' g:'+col_g+' _pbar_warn:'+ _pbar_warn + ' color: '+col); + element_id("gradient").style.backgroundColor = col; + + element_id("progressIndicator").innerHTML = pcnt + '%'; + //element_id("progressIndicator").innerHTML = amount; + + element_id("mask").style.clip = 'rect(' + Array( + '0px', + element_id("mask").offsetWidth + 'px', + element_id("mask").offsetHeight + 'px', + Math.round(_pbar_width * amount / max) + 'px' + ).join(' ') + ')'; +} + +if (!self.body) self.body = new Object(); +self.onload = self.document.onload = self.body.onload = function() { + //pbar_reset(); + sumiraj(); +}; + +// --> + +
+ + +Size: kB + +
+
 
+
 
+
0%
+
+
+ +
 
+ +Note: + + + + +
+ EOF3 $retHTML .= q{ - - - + } . + sort_header($param, 'Share', 'share', 'center') . + sort_header($param, '#', 'num', 'center') . + qq{ - - - - - - - + } . + sort_header($param, 'Date', 'date', 'center') . + sort_header($param, 'Age/days', 'age', 'center') . + sort_header($param, 'Size/Mb', 'size', 'center') . + sort_header($param, 'gzip size/Kb', 'incsize', 'center') . + qq{ + }; my @color = (' bgcolor="#e0e0e0"', ''); @@ -415,38 +902,44 @@ EOF3 my $i = 0; my $host = ''; - foreach my $backup ( getBackupsNotBurned() ) { + foreach my $backup ( getBackupsNotBurned($param) ) { if ($host ne $backup->{'host'}) { $i++; $host = $backup->{'host'}; } my $ftype = ""; - - $retHTML .= ""; - $retHTML .= ''; - + + my $checkbox_key = $backup->{'hostid'}. '_' .$backup->{'backupnum'} . '_' . $backup->{'id'}; + $retHTML .= + ' + ' . '' . '' . '' . '' . '' . '' . - '\n"; - + '' . + '' . + '' . + "\n"; } $retHTML .= "
- + ShareBackup noTypedateage/dayssize/MBgzip size
- -
medias
'; + + if (($backup->{'inc_size'} || 0) > 0) { + $retHTML .= ' + '; + } + + my $img_url = $Conf{CgiImageDirURL}; + + $retHTML .= + '' . $backup->{'host'} . ':' . $backup->{'share'} . '' . $backup->{'backupnum'} . '' . $backup->{'type'} . '' . epoch_to_iso( $backup->{'date'} ) . '' . $backup->{'age'} . '' . $backup->{'size'} . '' . $backup->{'fs_size'} . - '' . - "
' . sprintf("%0.1f", $backup->{'inc_size'} / 1024 ) . + '' . ( qq{media} x $backup->{volumes} ) . '
"; - $retHTML .= "total gzip size:
"; - $retHTML .= "Note:"; $retHTML .= "
"; return $retHTML; @@ -493,11 +986,53 @@ sub displayGrid($) { - - - - - + }; + + sub sort_header($$$$) { + my ($param, $display, $name, $align) = @_; + + my ($sort_what, $sort_direction) = split(/_/,$param->{'sort'},2); + + my $old_sort = $param->{'sort'}; + + my $html = qq{'; + $param->{'sort'} = $old_sort; + + return $html; + } + + $retHTML .= + sort_header($param, 'Share', 'share', 'center') . + sort_header($param, 'Type and Name', 'path', 'center') . + sort_header($param, '#', 'num', 'center') . + sort_header($param, 'Size', 'size', 'center') . + sort_header($param, 'Date', 'date', 'center'); + + $retHTML .= qq{ }; @@ -517,6 +1052,46 @@ sub displayGrid($) { return sprintf(qq{%s}, $action, @_); } + my $sth_archived; + my %archived_cache; + + sub check_archived($$$) { + my ($host, $share, $num) = @_; + + if (my $html = $archived_cache{"$host $share $num"}) { + return $html; + } + + $sth_archived ||= $dbh->prepare(qq{ + select + dvd_nr, note, + count(archive_burned.copy) as copies + from archive + inner join archive_burned on archive_burned.archive_id = archive.id + inner join archive_backup on archive.id = archive_backup.archive_id + inner join backups on backups.id = archive_backup.backup_id + inner join hosts on hosts.id = backups.hostid + inner join shares on shares.id = backups.shareid + where hosts.name = ? and shares.name = ? and backups.num = ? + group by dvd_nr, note + }); + + my @mediums; + + $sth_archived->execute($host, $share, $num); + while (my $row = $sth_archived->fetchrow_hashref()) { + push @mediums, '' .$row->{'dvd_nr'} . + ''; + } + + my $html = join(", ",@mediums); + $archived_cache{"$host $share $num"} = $html; + return $html; + } + my $i = $offset * $on_page; foreach $file (@{ $files }) { @@ -533,7 +1108,7 @@ sub displayGrid($) { qq{} . qq{} . qq{} . - qq{}; + qq{}; $retHTML .= ""; } @@ -548,20 +1123,26 @@ sub displayGrid($) { my $max_page = int( $results / $on_page ); my $page = 0; - sub page_link($$$) { - my ($param,$page,$display) = @_; - - $param->{'offset'} = $page; + sub page_uri($) { + my $param = shift || die "no param?"; - my $html = '' . $display . ''; + return $uri; + } + + sub page_link($$$) { + my ($param,$page,$display) = @_; + + $param->{'offset'} = $page if (defined($page)); + + my $html = '' . $display . ''; } $retHTML .= '
';
ShareType and Name#SizeDate{'sort'} = $name . '_' . $direction; + $html .= ' style="border: 1px solid #808080;"'; + + # add unicode arrow for direction + $arrow .= ' '; + $arrow .= $direction eq 'a' ? '▲' + : $direction eq 'd' ? '▼' + : '' + ; + + } else { + $param->{'sort'} = $name . '_a'; + } + + $html .= '>' . $display . '' . $arrow . 'Media
} . restore_link( $typeStr, ${EscURI( $file->{'hname'} )}, $file->{'backupnum'}, ${EscURI( $file->{'sname'})}, ${EscURI( $file->{'filepath'} )}, $file->{'backupnum'} ) . qq{} . $file->{'size'} . qq{} . epoch_to_iso( $file->{'date'} ) . qq{} . '?' . qq{} . check_archived( $file->{'hname'}, $file->{'sname'}, $file->{'backupnum'} ) . qq{