Bug 18300: Delete missing upload records

If you deleted files from the upload directories manually, or you rebooted
with files in the temporary upload folder, or for some other reason have
records without a file, you may want to cleanup your records in the
uploaded_files table.

This patch adds the method delete_missing to Koha::UploadedFiles. It also
supports a keep_record parameter to do a dry run (count the missing files
first).

Also, we add an option --uploads-missing to cleanup_database. If you add
the flag 1 after this option, you will delete missing files. If you add the
flag 0 or only use the option, you will count missing files.

A subtest is added to Upload.t for delete_missing tests.

Test plan:
[1] Run t/db_dependent/Upload.t
[2] Upload a file and delete the file manually.
[3] Run cleanup_database.pl --uploads-missing
    It should report at least one missing file now.
    Check that the record has not been deleted.
[4] Run cleanup_database.pl --uploads-missing 1
    It should report that it removed at least one file.
    Check that the record is gone.

Signed-off-by: Marcel de Rooy <m.de.rooy@rijksmuseum.nl>
Signed-off-by: Mirko Tietgen <mirko@abunchofthings.net>
Signed-off-by: Jonathan Druart <jonathan.druart@bugs.koha-community.org>
Signed-off-by: Kyle M Hall <kyle@bywatersolutions.com>

diff --git a/Koha/UploadedFiles.pm b/Koha/UploadedFiles.pm
index b64df0a..61e9702 100644 (file)
--- a/Koha/UploadedFiles.pm
+++ b/Koha/UploadedFiles.pm
@@ -105,6 +105,36 @@ sub delete_temporary {
     })->delete;
 }
 
+=head3 delete_missing
+
+    $cnt = Koha::UploadedFiles->delete_missing();
+
+    $cnt = Koha::UploadedFiles->delete_missing({ keep_record => 1 });
+
+Deletes all records where the actual file is not found.
+
+Supports a keep_record hash parameter to do a check only.
+
+Returns the number of missing files (and/or deleted records).
+
+=cut
+
+sub delete_missing {
+    my ( $self, $params ) = @_;
+    $self = Koha::UploadedFiles->new if !ref($self); # handle class call
+    my $cnt = 0;
+    while( my $row = $self->next ) {
+        if( my $file = $row->full_path ) {
+            next if -e $file;
+            # We are passing keep_file since we already know that the file
+            # is missing and we do not want to see the warning
+            $row->delete({ keep_file => 1 }) if !$params->{keep_record};
+            $cnt++;
+        }
+    }
+    return $cnt;
+}
+
 =head3 search_term
 
 Search_term allows you to pass a term to search in filename and hashvalue.

diff --git a/misc/cronjobs/cleanup_database.pl b/misc/cronjobs/cleanup_database.pl
index b39d9a7..03c382d 100755 (executable)
--- a/misc/cronjobs/cleanup_database.pl
+++ b/misc/cronjobs/cleanup_database.pl
@@ -44,7 +44,7 @@ use Koha::UploadedFiles;
 
 sub usage {
     print STDERR <<USAGE;
-Usage: $0 [-h|--help] [--sessions] [--sessdays DAYS] [-v|--verbose] [--zebraqueue DAYS] [-m|--mail] [--merged] [--import DAYS] [--logs DAYS] [--searchhistory DAYS] [--restrictions DAYS] [--all-restrictions] [--fees DAYS] [--temp-uploads] [--temp-uploads-days DAYS]
+Usage: $0 [-h|--help] [--sessions] [--sessdays DAYS] [-v|--verbose] [--zebraqueue DAYS] [-m|--mail] [--merged] [--import DAYS] [--logs DAYS] [--searchhistory DAYS] [--restrictions DAYS] [--all-restrictions] [--fees DAYS] [--temp-uploads] [--temp-uploads-days DAYS] [--uploads-missing 0|1 ]
 
    -h --help          prints this help message, and exits, ignoring all
                       other options
@@ -83,6 +83,7 @@ Usage: $0 [-h|--help] [--sessions] [--sessdays DAYS] [-v|--verbose] [--zebraqueu
    --unique-holidays DAYS  Delete all unique holidays older than DAYS
    --temp-uploads     Delete temporary uploads.
    --temp-uploads-days DAYS Override the corresponding preference value.
+   --uploads-missing FLAG Delete upload records for missing files when FLAG is true, count them otherwise
 USAGE
     exit $_[0];
 }
@@ -107,6 +108,7 @@ my $fees_days;
 my $special_holidays_days;
 my $temp_uploads;
 my $temp_uploads_days;
+my $uploads_missing;
 
 GetOptions(
     'h|help'            => \$help,
@@ -129,6 +131,7 @@ GetOptions(
     'unique-holidays:i' => \$special_holidays_days,
     'temp-uploads'      => \$temp_uploads,
     'temp-uploads-days:i' => \$temp_uploads_days,
+    'uploads-missing:i' => \$uploads_missing,
 ) || usage(1);
 
 # Use default values
@@ -161,6 +164,7 @@ unless ( $sessions
     || $pUnvSelfReg
     || $special_holidays_days
     || $temp_uploads
+    || defined $uploads_missing
 ) {
     print "You did not specify any cleanup work for the script to do.\n\n";
     usage(1);
@@ -321,6 +325,17 @@ if( $temp_uploads ) {
     print "Done purging temporary uploads.\n" if $verbose;
 }
 
+if( defined $uploads_missing ) {
+    print "Looking for missing uploads\n" if $verbose;
+    my $keep = $uploads_missing == 1 ? 0 : 1;
+    my $count = Koha::UploadedFiles->delete_missing({ keep_record => $keep });
+    if( $keep ) {
+        print "Counted $count missing uploaded files\n";
+    } else {
+        print "Removed $count records for missing uploads\n";
+    }
+}
+
 exit(0);
 
 sub RemoveOldSessions {

diff --git a/t/db_dependent/Upload.t b/t/db_dependent/Upload.t
index 8f6add2..0489fee 100644 (file)
--- a/t/db_dependent/Upload.t
+++ b/t/db_dependent/Upload.t
@@ -2,7 +2,7 @@
 
 use Modern::Perl;
 use File::Temp qw/ tempdir /;
-use Test::More tests => 11;
+use Test::More tests => 12;
 use Test::Warn;
 
 use Test::MockModule;
@@ -18,7 +18,7 @@ use Koha::Uploader;
 
 my $schema  = Koha::Database->new->schema;
 $schema->storage->txn_begin;
-my $builder = t::lib::TestBuilder->new;
+our $builder = t::lib::TestBuilder->new;
 
 our $current_upload = 0;
 our $uploads = [
@@ -192,6 +192,23 @@ subtest 'Test delete via UploadedFile as well as UploadedFiles' => sub {
     # NOTE: Koha::Object->delete does not return 0E0 (yet?)
 };
 
+subtest 'Test delete_missing' => sub {
+    plan tests => 4;
+
+    # If we add files via TestBuilder, they do not exist
+    my $upload01 = $builder->build({ source => 'UploadedFile' });
+    my $upload02 = $builder->build({ source => 'UploadedFile' });
+    # dry run first
+    my $deleted = Koha::UploadedFiles->delete_missing({ keep_record => 1 });
+    is( $deleted, 2, 'Expect two missing files' );
+    isnt( Koha::UploadedFiles->find( $upload01->{id} ), undef, 'Not deleted' );
+    $deleted = Koha::UploadedFiles->delete_missing;
+    is( $deleted, 2, 'Deleted two missing files' );
+    is( Koha::UploadedFiles->search({
+        id => [ $upload01->{id}, $upload02->{id} ],
+    })->count, 0, 'Records are gone' );
+};
+
 subtest 'Call search_term with[out] private flag' => sub {
     plan tests => 3;