Bug 7977: QOTD uploader to enable uploading csv files containing quotes
authorChris Nighswonger <cnighswonger@foundations.edu>
Fri, 20 Apr 2012 20:12:26 +0000 (16:12 -0400)
committerPaul Poulain <paul.poulain@biblibre.com>
Thu, 24 May 2012 12:14:09 +0000 (14:14 +0200)
This series will add a DataTable's based upload/editor with which
to upload csv files containing quotes to be used by the QOTD
feature.

The file should be formatted thusly:

"source","text-of-quote"
"source","text-of-quote"
...

Note: This work serves as a good example of potential improvements
in all other "editor" and file upload areas of Koha.

This patch is a squash of the following work:

--Adding code to parse CSV file contents and push it into a DataTable
--Adding in jEditable to enable table editing
--Adding ajax to post data back to the server to be saved
--Fixing edit and adding delete functionality
--Adding some missing css as well as server feedback on save
--Fixing a bug which limited the number of quotes which could be uploaded
--Also fixing a minor bug with fnCSVToArray and doing some style cleanup.
--Adding sanity checks to verify file type and size
--Implements YUI button widget/toolbar
--Improved handling of hiding uploader UI
--Adds row selectability
--Adds multi-delete capability
--Adds YUI button/toolbar widget
--Fixing capitalization in quote uploader
--Implements improvments suggested by jcamins and oleonard

Signed-off-by: Jared Camins-Esakov <jcamins@cpbibliography.com>
Signed-off-by: Mason James <mtj@kohaaloha.com>
koha-tmpl/intranet-tmpl/prog/en/css/datatables.css
koha-tmpl/intranet-tmpl/prog/en/css/uploader.css [new file with mode: 0644]
koha-tmpl/intranet-tmpl/prog/en/includes/quotes-upload-toolbar.inc [new file with mode: 0644]
koha-tmpl/intranet-tmpl/prog/en/modules/tools/quotes-upload.tt [new file with mode: 0644]
koha-tmpl/intranet-tmpl/prog/en/modules/tools/quotes.tt
koha-tmpl/intranet-tmpl/prog/img/x_alt_16x16.png [new file with mode: 0644]
tools/quotes-upload.pl [new file with mode: 0755]
tools/quotes/quotes-upload_ajax.pl [new file with mode: 0755]

index 2a78678..a54c612 100644 (file)
@@ -191,6 +191,18 @@ div.dataTables_paginate.paging_four_button {
     width: 250px;
 }
 
+input {
+    border-radius: 5px;
+}
+
+tr.odd.selected td {
+    background-color: #D3D3D3;
+}
+
+tr.even.selected td {
+    background-color: #D3D3D3;
+}
+
 /*
 table.display {
     width: 100%;
diff --git a/koha-tmpl/intranet-tmpl/prog/en/css/uploader.css b/koha-tmpl/intranet-tmpl/prog/en/css/uploader.css
new file mode 100644 (file)
index 0000000..e747255
--- /dev/null
@@ -0,0 +1,49 @@
+#progress_bar {
+  margin: 10px 0;
+  padding: 3px;
+  border: 1px solid #000;
+  font-size: 14px;
+  clear: both;
+  opacity: 0;
+  -moz-transition: opacity 1s linear;
+  -o-transition: opacity 1s linear;
+  -webkit-transition: opacity 1s linear;
+}
+#progress_bar.loading {
+    opacity: 1.0;
+}
+#progress_bar .percent {
+    color: #E6F0F2;
+    background-color: #004D99;
+    height: auto;
+    width: 0;
+}
+#server_response {
+    background-color: white;
+    background-image: url("../../img/x_alt_16x16.png");
+    background-repeat: no-repeat;
+    background-origin: padding-box;
+    background-position: right top;
+    border: 1px solid #DDDDDD;
+    color: #999999;
+    font-size: 14px;
+    height: 30px;
+    left: 50%;
+    margin-left: -125px;
+    margin-top: -15px;
+    padding: 14px 0 2px;
+    position: fixed;
+    text-align: center;
+    top: 50%;
+    width: 250px;
+}
+
+#file_uploader {
+    position: relative;
+    top: -24px;
+}
+
+#file_upload {
+    position: relative;
+    top: -1px;
+}
diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/quotes-upload-toolbar.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/quotes-upload-toolbar.inc
new file mode 100644 (file)
index 0000000..f32e8b1
--- /dev/null
@@ -0,0 +1,42 @@
+<script type="text/javascript">
+    var oSaveButton = '';
+    var oDeleteButton = '';
+    //<![CDATA[
+    // prepare DOM for YUI Toolbar
+    $(document).ready(function() {
+        $("#save_quotes").empty();
+        $("#delete_quote").empty();
+        yuiToolbar();
+     });
+
+    // YUI Toolbar Functions
+    function yuiGetData() {
+        fnGetData(document.getElementById('quotes_editor'));
+    }
+
+    function yuiToolbar() {
+
+        oSaveButton = new YAHOO.widget.Button({
+            type: "button",
+            label: _("Save quotes"),
+            name: "save",
+            container: "save_quotes",
+        });
+
+        oDeleteButton = new YAHOO.widget.Button({
+            type: "button",
+            label: _("Delete quote(s)"),
+            name: "delete",
+            container: "delete_quote",
+        });
+    }
+
+//]]>
+</script>
+<div id="toolbar">
+    <ul class="toolbar">
+        <li id="save_quotes"><a id="save" href="#">Save quotes</a></li>
+        <li id="delete_quote"><a id="delete" href="#">Delete quote(s)</a></li>
+        <span class="hint" style="">Click Source or Text field to edit contents. Press &lt;Enter&gt; to save changes.</span>
+    </ul>
+</div>
diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/quotes-upload.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/quotes-upload.tt
new file mode 100644 (file)
index 0000000..f2cb169
--- /dev/null
@@ -0,0 +1,337 @@
+    [% INCLUDE 'doc-head-open.inc' %]
+    <title>Koha &rsaquo; Tools &rsaquo; Quote uploader</title>
+    [% INCLUDE 'doc-head-close.inc' %]
+    <link rel="stylesheet" type="text/css" href="/intranet-tmpl/prog/en/css/uploader.css" />
+    <link rel="stylesheet" type="text/css" href="/intranet-tmpl/prog/en/css/datatables.css" />
+    <script type="text/javascript" src="/intranet-tmpl/prog/en/lib/jquery/plugins/jquery.dataTables.min.js"></script>
+    [% INCLUDE 'datatables-strings.inc' %]
+    </script>
+    <script type="text/javascript" src="/intranet-tmpl/prog/en/js/datatables.js"></script>
+    <script type="text/javascript" src="/intranet-tmpl/prog/en/js/jquery.jeditable.mini.js"></script>
+    <script type="text/javascript">
+    //<![CDATA[
+    var oTable; //DataTable object
+    $(document).ready(function() {
+
+    // Credits:
+    // FileReader() code copied and hacked from:
+    // http://www.html5rocks.com/en/tutorials/file/dndfiles/
+    // fnCSVToArray() gratefully borrowed from:
+    // http://www.bennadel.com/blog/1504-Ask-Ben-Parsing-CSV-Strings-With-Javascript-Exec-Regular-Expression-Command.htm
+
+    var reader;
+    var progress = document.querySelector('.percent');
+    $("#server_response").hide();
+
+    function fnAbortRead() {
+        reader.abort();
+    }
+
+    function fnErrorHandler(evt) {
+        switch(evt.target.error.code) {
+            case evt.target.error.NOT_FOUND_ERR:
+                alert('File Not Found!');
+                break;
+            case evt.target.error.NOT_READABLE_ERR:
+                alert('File is not readable');
+                break;
+            case evt.target.error.ABORT_ERR:
+                break; // noop
+            default:
+                alert('An error occurred reading this file.');
+        };
+    }
+
+    function fnUpdateProgress(evt) {
+        // evt is an ProgressEvent.
+        if (evt.lengthComputable) {
+            var percentLoaded = Math.round((evt.loaded / evt.total) * 100);
+            // Increase the progress bar length.
+            if (percentLoaded < 100) {
+                progress.style.width = percentLoaded + '%';
+                progress.textContent = percentLoaded + '%';
+            }
+        }
+    }
+
+    function fnCSVToArray( strData, strDelimiter ){
+        // This will parse a delimited string into an array of
+        // arrays. The default delimiter is the comma, but this
+        // can be overriden in the second argument.
+
+        // Check to see if the delimiter is defined. If not,
+        // then default to comma.
+        strDelimiter = (strDelimiter || ",");
+
+        // Create a regular expression to parse the CSV values.
+        var objPattern = new RegExp(
+        (
+            // Delimiters.
+            "(\\" + strDelimiter + "|\\r?\\n|\\r|^)" +
+            // Quoted fields.
+            "(?:\"([^\"]*(?:\"\"[^\"]*)*)\"|" +
+            // Standard fields.
+            "([^\"\\" + strDelimiter + "\\r\\n]*))"
+        ),
+            "gi"
+        );
+
+        // Create an array to hold our data. Give the array
+        // a default empty first row.
+        var arrData = [[]];
+
+        // Create an array to hold our individual pattern
+        // matching groups.
+        var arrMatches = null;
+
+        // Keep looping over the regular expression matches
+        // until we can no longer find a match.
+        while (arrMatches = objPattern.exec( strData )){
+
+            // Get the delimiter that was found.
+            var strMatchedDelimiter = arrMatches[ 1 ];
+
+            // Check to see if the given delimiter has a length
+            // (is not the start of string) and if it matches
+            // field delimiter. If it does not, then we know
+            // that this delimiter is a row delimiter.
+            if ( strMatchedDelimiter.length && (strMatchedDelimiter != strDelimiter) ){
+                // Since we have reached a new row of data,
+                // add an empty row to our data array.
+                // Note: if there is not more data, we will have to remove this row later
+                arrData.push( [] );
+            }
+
+            // Now that we have our delimiter out of the way,
+            // let's check to see which kind of value we
+            // captured (quoted or unquoted).
+            if (arrMatches[ 2 ]){
+                // We found a quoted value. When we capture
+                // this value, unescape any double quotes.
+                var strMatchedValue = arrMatches[ 2 ].replace(
+                new RegExp( "\"\"", "g" ),
+                    "\""
+                );
+            } else if (arrMatches[3]){
+                // We found a non-quoted value.
+                var strMatchedValue = arrMatches[ 3 ];
+            } else {
+                // There is no more valid data so remove the row we added earlier
+                // Is there a better way? Perhaps a look-ahead regexp?
+                arrData.splice(arrData.length-1, 1);
+            }
+
+            // Now that we have our value string, let's add
+            // it to the data array.
+            arrData[ arrData.length - 1 ].push( strMatchedValue );
+        }
+
+        // Return the parsed data.
+        return( arrData );
+    }
+
+    function fnDataTable(aaData) {
+        for(var i=0; i<aaData.length; i++) {
+            aaData[i].unshift(i+1); // Add a column w/quote number
+        }
+        $('#save_quotes').css("visibility","visible");
+        $('#file_uploader').css("visibility","hidden");
+        $('#file_uploader').css("position","absolute");
+        $('#file_uploader').css("top","-150px");
+        $('#quotes_editor').css("visibility","visible");
+        oSaveButton.on("click", yuiGetData);
+        oDeleteButton.on("click", fnClickDeleteRow);
+        oTable = $('#quotes_editor').dataTable( {
+            "bAutoWidth"        : false,
+            "bPaginate"         : true,
+            "bSort"             : false,
+            "sPaginationType"   : "full_numbers",
+            "aaData"            : aaData,
+            "aoColumns"         : [
+                {
+                    "sTitle"  : "Number",
+                    "sWidth"  : "2%",
+                },
+                {
+                    "sTitle"  : "Source",
+                    "sWidth"  : "15%",
+                },
+                {
+                    "sTitle"  : "Quote",
+                    "sWidth"  : "83%",
+                },
+            ],
+           "fnPreDrawCallback": function(oSettings) {
+                return true;
+            },
+            "fnRowCallback": function( nRow, aData, iDisplayIndex ) {
+                /* do foo on various cells in the current row */
+                var quoteNum = $('td', nRow)[0].innerHTML;
+                $(nRow).attr("id", quoteNum); /* set row ids to quote number */
+                $('td:eq(0)', nRow).click(function() {$(this.parentNode).toggleClass('selected',this.clicked);}); /* add row selectors */
+                $('td:eq(0)', nRow).attr("title", "Click ID to select/deselect quote");
+                /* apply no_edit id to noEditFields */
+                noEditFields = [0]; /* number */
+                for (i=0; i<noEditFields.length; i++) {
+                    $('td', nRow)[noEditFields[i]].setAttribute("id","no_edit");
+                }
+                return nRow;
+            },
+           "fnDrawCallback": function(oSettings) {
+                /* Apply the jEditable handlers to the table on all fields w/o the no_edit id */
+                $('#quotes_editor tbody td[id!="no_edit"]').editable( function(value, settings) {
+                        var cellPosition = oTable.fnGetPosition( this );
+                        oTable.fnUpdate(value, cellPosition[0], cellPosition[1], false, false);
+                        return(value);
+                    },
+                    {
+                    "callback"      : function( sValue, y ) {
+                                          oTable.fnDraw(false); /* no filter/sort or we lose our pagination */
+                                      },
+                    "height"        : "14px",
+                });
+           },
+        });
+        $('#footer').css("visibility","visible");
+    }
+
+    function fnHandleFileSelect(evt) {
+        // Reset progress indicator on new file selection.
+        progress.style.width = '0%';
+        progress.textContent = '0%';
+
+        reader = new FileReader();
+        reader.onerror = fnErrorHandler;
+        reader.onprogress = fnUpdateProgress;
+        reader.onabort = function(e) {
+            alert('File read cancelled');
+            parent.location='quotes-upload.pl';
+        };
+        reader.onloadstart = function(e) {
+            $('#cancel_upload').css("visibility","visible");
+            $('#progress_bar').addClass("loading");
+        };
+        reader.onload = function(e) {
+            // Ensure that the progress bar displays 100% at the end.
+            progress.style.width = '100%';
+            progress.textContent = '100%';
+            $('#cancel_upload').css("visibility","hidden");
+            quotes = fnCSVToArray(e.target.result, ',');
+            fnDataTable(quotes);
+        }
+
+        // perform various sanity checks on the target file prior to uploading...
+        var fileType = evt.target.files[0].type || 'unknown';
+        var fileSizeInK = Math.round(evt.target.files[0].size/1024);
+
+        if (!fileType.match(/comma-separated-values|csv|excel/i)) {
+            alert('Incorrect filetype: '+fileType+'. Uploads limited to csv.');
+            parent.location='quotes-upload.pl';
+            return;
+        }
+        if (fileSizeInK > 512) {
+            if (!confirm(evt.target.files[0].name+' is '+fileSizeInK+' K in size. Do you really want to upload this file?')) {
+                parent.location='quotes-upload.pl';
+                return;
+            }
+        }
+        // Read in the image file as a text string.
+        reader.readAsText(evt.target.files[0]);
+    }
+
+    $('#file_upload').one('change', fnHandleFileSelect);
+
+    });
+
+    function fnGetData(element) {
+        var jqXHR = $.ajax({
+            url         : "/cgi-bin/koha/tools/quotes/quotes-upload_ajax.pl",
+            type        : "POST",
+            contentType : "application/x-www-form-urlencoded", // we must claim this mimetype or CGI will not decode the URL encoding
+            dataType    : "json",
+            data        : {
+                            "quote"     : JSON.stringify(oTable.fnGetData()),
+                            "action"    : "add",
+                          },
+            success     : function(){
+                            var response = JSON.parse(jqXHR.responseText);
+                            if (response.success) {
+                                $("#server_response").text(response.records+' quotes saved.');
+                            }
+                            else {
+                                $("#server_response").text('An error has occurred. '+response.records+' quotes saved. Please ask your administrator to check the server log for more details.');
+                            }
+                            $("#server_response").fadeIn(200);
+                          },
+        });
+    }
+
+    function fnClickDeleteRow() {
+        var idsToDelete = oTable.$('.selected').map(function() {
+              return this.id;
+        }).get().join(', ');
+        if (!idsToDelete) {
+            alert('Please select a quote(s) by clicking the quote id(s) you desire to delete.');
+        }
+        else if (confirm('Are you sure you wish to delete quote(s) '+idsToDelete+'?')) {
+            oTable.$('.selected').each(function(){
+                oTable.fnDeleteRow(this);
+            });
+        }
+    }
+
+    function fnResetUpload() {
+        $('#server_response').fadeOut(200);
+        window.location.reload(true);   // is this the best route?
+    }
+
+    //]]>
+    </script>
+</head>
+<body id="tools_quotes" class="tools">
+[% INCLUDE 'header.inc' %]
+[% INCLUDE 'cat-search.inc' %]
+
+<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; <a href="/cgi-bin/koha/tools/quotes.pl">Quote editor</a> &rsaquo; Quote uploader</div>
+
+<div id="doc3" class="yui-t2">
+    <div id="bd">
+        <div id="yui-main">
+            <div class="yui-b">
+                [% INCLUDE 'quotes-upload-toolbar.inc' %]
+                <h2>Quote uploader</h2>
+                <fieldset id="file_uploader" class="rows" style="visibility:visible;">
+                    <legend>Upload quotes</legend>
+                    <div id="file_upload" style="margin-left: 10px;">
+                        <input type="file" name="file" />
+                        <button id="cancel_upload" style="visibility:hidden;" onclick="fnAbortRead();">Cancel Upload</button>
+                        <div id="progress_bar"><div class="percent">0%</div></div>
+                    </div>
+                </fieldset>
+                <div id="server_response" onclick='fnResetUpload()'>Server Response</div>
+                <table id="quotes_editor" style="float: left; width: 100%; visibility:hidden;">
+                <thead>
+                    <tr>
+                        <th>Source</th>
+                        <th>Text</th>
+                        <th>Actions</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    <!-- tbody content is generated by DataTables -->
+                    <tr>
+                        <td></td>
+                        <td>Loading data...</td>
+                        <td></td>
+                    </tr>
+                </tbody>
+                </table>
+                <fieldset id="footer" class="action" style="visibility:hidden; height:25px">
+                </fieldset>
+            </div>
+        </div>
+    <div class="yui-b noprint">
+        [% INCLUDE 'tools-menu.inc' %]
+    </div>
+</div>
+[% INCLUDE 'intranet-bottom.inc' %]
index 7d020b5..770bd31 100644 (file)
@@ -6,8 +6,8 @@
     <script type="text/javascript" src="[% themelang %]/lib/jquery/plugins/dataTables.fnReloadAjax.js"></script>
     [% INCLUDE 'datatables-strings.inc' %]
     </script>
-    <script type="text/javascript" src="/intranet-tmpl/prog/en/js/datatables.js"></script>
-    <script type="text/javascript" src="/intranet-tmpl/prog/en/js/jquery.jeditable.mini.js"></script>
+    <script type="text/javascript" src="[% themelang %]/js/datatables.js"></script>
+    <script type="text/javascript" src="[% themelang %]/js/jquery.jeditable.mini.js"></script>
     <script type="text/javascript">
     //<![CDATA[
     var oTable; /* oTable needs to be global */
diff --git a/koha-tmpl/intranet-tmpl/prog/img/x_alt_16x16.png b/koha-tmpl/intranet-tmpl/prog/img/x_alt_16x16.png
new file mode 100644 (file)
index 0000000..a99310e
Binary files /dev/null and b/koha-tmpl/intranet-tmpl/prog/img/x_alt_16x16.png differ
diff --git a/tools/quotes-upload.pl b/tools/quotes-upload.pl
new file mode 100755 (executable)
index 0000000..7db61ce
--- /dev/null
@@ -0,0 +1,44 @@
+#!/usr/bin/perl
+
+# Copyright 2012 Foundations Bible College Inc.
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+use strict;
+use warnings;
+
+use CGI;
+use autouse 'Data::Dumper' => qw(Dumper);
+
+use C4::Auth;
+use C4::Koha;
+use C4::Context;
+use C4::Output;
+
+my $cgi = new CGI;
+
+my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
+    {
+        template_name   => "tools/quotes-upload.tt",
+        query           => $cgi,
+        type            => "intranet",
+        authnotrequired => 0,
+        flagsrequired   => { tools => 'edit_quotes' },
+        debug           => 1,
+    }
+);
+
+output_html_with_http_headers $cgi, $cookie, $template->output;
diff --git a/tools/quotes/quotes-upload_ajax.pl b/tools/quotes/quotes-upload_ajax.pl
new file mode 100755 (executable)
index 0000000..f1c3746
--- /dev/null
@@ -0,0 +1,68 @@
+#!/usr/bin/perl
+
+# Copyright 2012 Foundations Bible College Inc.
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Koha; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+use strict;
+use warnings;
+
+use CGI;
+use JSON;
+use autouse 'Data::Dumper' => qw(Dumper);
+
+use C4::Auth;
+use C4::Koha;
+use C4::Context;
+use C4::Output;
+
+my $cgi = new CGI;
+my $dbh = C4::Context->dbh;
+
+my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
+    {
+        template_name   => "",
+        query           => $cgi,
+        type            => "intranet",
+        authnotrequired => 0,
+        flagsrequired   => { tools => 'edit_quotes' },
+        debug           => 1,
+    }
+);
+
+my $success = 'true';
+
+my $quotes = decode_json($cgi->param('quote'));
+my $action = $cgi->param('action');
+
+my $sth = $dbh->prepare('INSERT INTO quotes (source, text) VALUES (?, ?);');
+
+my $insert_count = 0;
+
+foreach my $quote (@$quotes) {
+    $insert_count++ if $sth->execute($quote->[0], $quote->[1]);
+    if ($sth->err) {
+        warn sprintf('Database returned the following error: %s', $sth->errstr);
+        $success = 'false';
+    }
+}
+
+print $cgi->header('application/json');
+
+print to_json({
+                success => $success,
+                records => $insert_count,
+});