Bug 14356: Improvements to the 'Transfers to receive' page
[koha.git] / tools / export.pl
index a4eb6e9..f63e2c4 100755 (executable)
@@ -3,26 +3,27 @@
 #
 # This file is part of Koha.
 #
-# Koha is free software; you can redistribute it and/or modify it under the
-# terms of the GNU General Public License as published by the Free Software
-# Foundation; either version 2 of the License, or (at your option) any later
-# version.
+# Koha is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
 #
-# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
-# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
-# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+# Koha is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License along with
-# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
-# Suite 330, Boston, MA  02111-1307 USA
+# You should have received a copy of the GNU General Public License
+# along with Koha; if not, see <http://www.gnu.org/licenses>.
 
 use Modern::Perl;
+use MARC::File::XML;
 use List::MoreUtils qw(uniq);
 use Getopt::Long;
-use CGI;
+use CGI qw ( -utf8 );
 use C4::Auth;
 use C4::AuthoritiesMarc;    # GetAuthority
-use C4::Biblio;             # GetMarcBiblio GetXmlBiblio
+use C4::Biblio;             # GetMarcBiblio
 use C4::Branch;             # GetBranches
 use C4::Csv;
 use C4::Koha;               # GetItemTypes
@@ -32,21 +33,25 @@ use C4::Record;
 my $query = new CGI;
 
 my $clean;
-my $output_format;
 my $dont_export_items;
 my $deleted_barcodes;
 my $timestamp;
 my $record_type;
+my $id_list_file;
 my $help;
 my $op       = $query->param("op")       || '';
 my $filename = $query->param("filename") || 'koha.mrc';
 my $dbh      = C4::Context->dbh;
 my $marcflavour = C4::Context->preference("marcflavour");
-my $format = $query->param("format") || 'iso2709';
+my $output_format = $query->param("format") || $query->param("output_format") || 'iso2709';
 
 # Checks if the script is called from commandline
 my $commandline = not defined $ENV{GATEWAY_INTERFACE};
 
+
+# @biblionumbers is only use for csv export from circulation.pl
+my @biblionumbers = uniq $query->param("biblionumbers");
+
 if ( $commandline ) {
 
     # Getting parameters
@@ -59,12 +64,13 @@ if ( $commandline ) {
         'clean'             => \$clean,
         'filename=s'        => \$filename,
         'record-type=s'     => \$record_type,
+        'id_list_file=s'    => \$id_list_file,
         'help|?'            => \$help
     );
 
     if ($help) {
         print <<_USAGE_;
-export.pl [--format=format] [--date=date] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] --filename=outputfile
+export.pl [--format=format] [--date=date] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
 
 
  --format=FORMAT        FORMAT is either 'xml' or 'marc' (default)
@@ -81,17 +87,22 @@ export.pl [--format=format] [--date=date] [--record-type=TYPE] [--dont_export_it
                         specified). Used only if TYPE is 'bibs'
 
  --clean                removes NSE/NSB
+
+ --id_list_file=PATH    PATH is a path to a file containing a list of
+                        IDs (biblionumber or authid) with one ID per line.
+                        This list works as a filter; it is compatible with
+                        other parameters for selecting records
 _USAGE_
         exit;
     }
 
     # Default parameters values :
-    $output_format     ||= 'marc';
     $timestamp         ||= '';
     $dont_export_items ||= 0;
     $deleted_barcodes  ||= 0;
     $clean             ||= 0;
     $record_type       ||= "bibs";
+    $id_list_file       ||= 0;
 
     # Redirect stdout
     open STDOUT, '>', $filename if $filename;
@@ -105,9 +116,14 @@ else {
 
 }
 
+# Default value for output_format is 'iso2709'
+$output_format ||= 'iso2709';
+# Retrocompatibility for the format parameter
+$output_format = 'iso2709' if $output_format eq 'marc';
+
 my ( $template, $loggedinuser, $cookie, $flags ) = get_template_and_user(
     {
-        template_name   => "tools/export.tmpl",
+        template_name   => "tools/export.tt",
         query           => $query,
         type            => "intranet",
         authnotrequired => $commandline,
@@ -117,26 +133,41 @@ my ( $template, $loggedinuser, $cookie, $flags ) = get_template_and_user(
 );
 
 my $limit_ind_branch =
-  (      C4::Context->preference('IndependantBranches')
+  (      C4::Context->preference('IndependentBranches')
       && C4::Context->userenv
-      && !( C4::Context->userenv->{flags} & 1 )
+      && !C4::Context->IsSuperLibrarian()
       && C4::Context->userenv->{branch} ) ? 1 : 0;
 
-my $branch = $query->param("branch") || '';
-if (   C4::Context->preference("IndependantBranches")
+my @branch = $query->param("branch");
+if (   C4::Context->preference("IndependentBranches")
     && C4::Context->userenv
-    && !( C4::Context->userenv->{flags} & 1 ) )
+    && !C4::Context->IsSuperLibrarian() )
 {
-    $branch = C4::Context->userenv->{'branch'};
+    @branch = ( C4::Context->userenv->{'branch'} );
 }
+# if stripping nonlocal items, use loggedinuser's branch
+my $localbranch = C4::Context->userenv->{'branch'};
+
+my %branchmap = map { $_ => 1 } @branch; # for quick lookups
 
 my $backupdir = C4::Context->config('backupdir');
 
 if ( $op eq "export" ) {
-    if ( $format eq "iso2709" or $format eq "xml" ) {
+    if (
+        $output_format eq "iso2709"
+            or $output_format eq "xml"
+            or (
+                $output_format eq 'csv'
+                    and not @biblionumbers
+            )
+    ) {
         my $charset  = 'utf-8';
         my $mimetype = 'application/octet-stream';
-        binmode STDOUT, ':encoding(UTF-8)';
+
+        binmode STDOUT, ':encoding(UTF-8)'
+            if $filename =~ m/\.gz$/
+                or $filename =~ m/\.bz2$/;
+
         if ( $filename =~ m/\.gz$/ ) {
             $mimetype = 'application/x-gzip';
             $charset  = '';
@@ -150,12 +181,10 @@ if ( $op eq "export" ) {
         print $query->header(
             -type       => $mimetype,
             -charset    => $charset,
-            -attachment => $filename
+            -attachment => $filename,
         ) unless ($commandline);
 
         $record_type = $query->param("record_type") unless ($commandline);
-        $output_format = $query->param("output_format") || 'marc'
-          unless ($commandline);
         my $export_remove_fields = $query->param("export_remove_fields");
         my @biblionumbers      = $query->param("biblionumbers");
         my @itemnumbers        = $query->param("itemnumbers");
@@ -195,6 +224,19 @@ if ( $op eq "export" ) {
         my $starting_authid = $query->param('starting_authid');
         my $ending_authid   = $query->param('ending_authid');
         my $authtype        = $query->param('authtype');
+        my $filefh;
+        if ($commandline) {
+            open $filefh,"<", $id_list_file or die "cannot open $id_list_file: $!" if $id_list_file;
+        } else {
+            $filefh = $query->upload("id_list_file");
+        }
+        my %id_filter;
+        if ($filefh) {
+            while (my $number=<$filefh>){
+                $number=~s/[\r\n]*$//;
+                $id_filter{$number}=1 if $number=~/^\d+$/;
+            }
+        }
 
         if ( $record_type eq 'bibs' and not @biblionumbers ) {
             if ($timestamp) {
@@ -223,7 +265,7 @@ if ( $op eq "export" ) {
                         itemstable           => $itemstable,
                         StartingBiblionumber => $StartingBiblionumber,
                         EndingBiblionumber   => $EndingBiblionumber,
-                        branch               => $branch,
+                        branch               => \@branch,
                         start_callnumber     => $start_callnumber,
                         end_callnumber       => $end_callnumber,
                         start_accession      => $start_accession,
@@ -300,15 +342,16 @@ if ( $op eq "export" ) {
             # Someone is trying to mess us up
             exit;
         }
-
         unless (@biblionumbers) {
             my $sth = $dbh->prepare($sql_query);
             $sth->execute(@sql_params);
             push @recordids, map {
                 map { $$_[0] } $_
             } @{ $sth->fetchall_arrayref };
+            @recordids = grep { exists($id_filter{$_}) } @recordids if scalar(%id_filter);
         }
 
+        my $xml_header_written = 0;
         for my $recordid ( uniq @recordids ) {
             if ($deleted_barcodes) {
                 my $q = "
@@ -339,14 +382,10 @@ if ( $op eq "export" ) {
                         my ( $homebranchfield, $homebranchsubfield ) =
                           GetMarcFromKohaField( 'items.homebranch', '' );
                         for my $itemfield ( $record->field($homebranchfield) ) {
-
-# if stripping nonlocal items, use loggedinuser's branch if they didn't select one
-                            $branch = C4::Context->userenv->{'branch'}
-                              unless $branch;
                             $record->delete_field($itemfield)
                               if ( $dont_export_items
-                                || $itemfield->subfield($homebranchsubfield) ne
-                                $branch );
+                                || $localbranch ne $itemfield->subfield(
+                                        $homebranchsubfield) );
                         }
                     }
                 }
@@ -356,45 +395,75 @@ if ( $op eq "export" ) {
                 }
 
                 if ($export_remove_fields) {
-                    my @fields = split " ", $export_remove_fields;
-                    foreach (@fields) {
-                        /^(\d*)(\w)?$/;
-                        my $field    = $1;
-                        my $subfield = $2;
-
-                        # skip if this record doesn't have this field
-                        next if not defined $record->field($field);
-                        if ($subfield) {
-                            $record->field($field)->delete_subfields($subfield);
-                        }
-                        else {
-                            $record->delete_field( $record->field($field) );
+                    for my $f ( split / /, $export_remove_fields ) {
+                        if ( $f =~ m/^(\d{3})(.)?$/ ) {
+                            my ( $field, $subfield ) = ( $1, $2 );
+
+                            # skip if this record doesn't have this field
+                            if ( defined $record->field($field) ) {
+                                if ( defined $subfield ) {
+                                    my @tags = $record->field($field);
+                                    foreach my $t (@tags) {
+                                        $t->delete_subfields($subfield);
+                                    }
+                                }
+                                else {
+                                    $record->delete_fields($record->field($field));
+                                }
+                            }
                         }
                     }
                 }
                 RemoveAllNsb($record) if ($clean);
                 if ( $output_format eq "xml" ) {
-                    if ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' )
-                    {
-                        print $record->as_xml_record('UNIMARCAUTH');
-                    }
-                    else {
-                        print $record->as_xml_record($marcflavour);
+                    unless ($xml_header_written) {
+                        MARC::File::XML->default_record_format(
+                            (
+                                     $marcflavour eq 'UNIMARC'
+                                  && $record_type eq 'auths'
+                            ) ? 'UNIMARCAUTH' : $marcflavour
+                        );
+                        print MARC::File::XML::header();
+                        print "\n";
+                        $xml_header_written = 1;
                     }
+                    print MARC::File::XML::record($record);
+                    print "\n";
                 }
-                else {
+                elsif ( $output_format eq 'iso2709' ) {
+                    my $errorcount_on_decode = eval { scalar(MARC::File::USMARC->decode( $record->as_usmarc )->warnings()) };
+                    if ($errorcount_on_decode or $@){
+                        warn $@ if $@;
+                        warn "record (number $recordid) is invalid and therefore not exported because its reopening generates warnings above";
+                        next;
+                    }
                     print $record->as_usmarc();
                 }
             }
         }
+        if ($xml_header_written) {
+            print MARC::File::XML::footer();
+            print "\n";
+        }
+        if ( $output_format eq 'csv' ) {
+            my $csv_profile_id = $query->param('csv_profile')
+                || GetCsvProfileId( C4::Context->preference('ExportWithCsvProfile') );
+            my $output =
+              marc2csv( \@recordids,
+                $csv_profile_id );
+
+            print $output;
+        }
+
         exit;
     }
-    elsif ( $format eq "csv" ) {
+    elsif ( $output_format eq "csv" ) {
         my @biblionumbers = uniq $query->param("biblionumbers");
         my @itemnumbers   = $query->param("itemnumbers");
+        my $csv_profile_id = $query->param('csv_profile') || GetCsvProfileId( C4::Context->preference('ExportWithCsvProfile') );
         my $output =
           marc2csv( \@biblionumbers,
-            GetCsvProfileId( C4::Context->preference('ExportWithCsvProfile') ),
+            $csv_profile_id,
             \@itemnumbers, );
         print $query->header(
             -type                        => 'application/octet-stream',
@@ -427,7 +496,7 @@ else {
         push @branchloop,
           {
             value      => $thisbranch,
-            selected   => $thisbranch eq $branch,
+            selected   => %branchmap ? $branchmap{$thisbranch} : 1,
             branchname => $branches->{$thisbranch}->{'branchname'},
           };
     }
@@ -466,9 +535,9 @@ else {
     $template->param(
         branchloop               => \@branchloop,
         itemtypeloop             => \@itemtypesloop,
-        DHTMLcalendar_dateformat => C4::Dates->DHTMLcalendar(),
         authtypeloop             => \@authtypesloop,
         export_remove_fields     => C4::Context->preference("ExportRemoveFields"),
+        csv_profiles             => C4::Csv::GetCsvProfiles('marc'),
     );
 
     output_html_with_http_headers $query, $cookie, $template->output;
@@ -503,14 +572,14 @@ sub construct_query {
             my $itemstable           = $params->{itemstable};
             my $StartingBiblionumber = $params->{StartingBiblionumber};
             my $EndingBiblionumber   = $params->{EndingBiblionumber};
-            my $branch               = $params->{branch};
+            my @branch               = @{ $params->{branch} };
             my $start_callnumber     = $params->{start_callnumber};
             my $end_callnumber       = $params->{end_callnumber};
-            my $start_accession      = $params->{star_accession};
+            my $start_accession      = $params->{start_accession};
             my $end_accession        = $params->{end_accession};
             my $itemtype             = $params->{itemtype};
             my $items_filter =
-                 $branch
+                 @branch
               || $start_callnumber
               || $end_callnumber
               || $start_accession
@@ -532,18 +601,18 @@ sub construct_query {
                 push @sql_params, $EndingBiblionumber;
             }
 
-            if ($branch) {
-                $sql_query .= " AND homebranch = ? ";
-                push @sql_params, $branch;
+            if (@branch) {
+                $sql_query .= " AND homebranch IN (".join(',',map({'?'} @branch)).")";
+                push @sql_params, @branch;
             }
 
             if ($start_callnumber) {
-                $sql_query .= " AND itemcallnumber <= ? ";
+                $sql_query .= " AND itemcallnumber >= ? ";
                 push @sql_params, $start_callnumber;
             }
 
             if ($end_callnumber) {
-                $sql_query .= " AND itemcallnumber >= ? ";
+                $sql_query .= " AND itemcallnumber <= ? ";
                 push @sql_params, $end_callnumber;
             }
             if ($start_accession) {