added command-line options

[mongodb-experiments.git] / dbi2mongo.pl

diff --git a/dbi2mongo.pl b/dbi2mongo.pl
index 369e930..7644a4a 100755 (executable)
--- a/dbi2mongo.pl
+++ b/dbi2mongo.pl
@@ -1,27 +1,32 @@
 #!/usr/bin/perl -w
 
 sub BEGIN {
-$ENV{DBI_AUTOPROXY}='dbi:Gofer:transport=stream;url=ssh:dpavlin@koha.ffzg.hr';
+#$ENV{DBI_AUTOPROXY}='dbi:Gofer:transport=stream;url=ssh:dpavlin@koha.ffzg.hr';
 }
 
 use strict;
 use DBI;
 use MongoDB;
 use Data::Dump qw/dump/;
+use Getopt::Long;
 
-$|++;
+my $debug = 0;
+my $drop = 0;
+my $limit = 10000;
 
-my $debug = @ARGV ? 1 : 0;
+GetOptions(
+       'debug!'  => \$debug,
+       'drop!'   => \$drop,
+       'limit=i' => \$limit,
+) || die $!;
 
 our ( $dbi, $user, $password ) = ( "DBI:mysql:database=test" );
-our ( $database, $collection ) = ( 'test', 'test' );
+our ( $database, $collection ) = ( '', '' );
 our ( $table,  $pk ) = ( 'biblio'      => 'biblionumber' );
 our ( $table2, $fk ) = ( 'biblioitems' => 'biblionumber' );
 
-my $limit      = 50000;
-my $join_limit = 10000;
-
-require 'config.pl';
+my $config = shift @ARGV || die "usage: $0 config.pl\n";
+require $config;
 
 warn "# $dbi $user -> $database $collection $table.$pk<->$table2.$fk\n";
 
@@ -33,7 +38,7 @@ my $dbh  = DBI->connect($dbi,$user,$password, {
 #      mysql_enable_utf8 => 1,
 });
 
-$db->drop if $debug;
+$db->drop if $drop;
 
 # db.items.find().sort({_id:-1}).limit(1);
 my $last = $coll->query()->sort({ '_id' => -1 })->limit(1)->next;
@@ -42,19 +47,38 @@ my $last_id = $last->{_id} || 0;
 
 print "import $table.$pk > $last_id from $dbi\n";
 
-my $sth = $dbh->prepare(qq{
-       select
-               $pk as _id,
-               $table.*
-       from $table
-       where $pk > ?
-       order by $pk asc
-       limit $limit
-});
+our $offset = 0;
+our $sth;
+
+sub select_table {
+
+       $sth = $dbh->prepare(qq{
+               select
+                       $pk as _id,
+                       $table.*
+               from $table
+               where $pk > ?
+               order by $pk asc
+               limit $limit
+               offset $offset
+       });
 
-$sth->execute( $last_id );
-warn "# $table columns ",dump( $sth->{NAME} );
-print "import ",$sth->rows," from $table\n";
+       print STDERR " $table:$offset ";
+       $sth->execute( $last_id );
+#      warn "# $table columns ",dump( $sth->{NAME} ) if $offset == 0;
+       print STDERR " join ", $sth->rows, " rows ";
+}
+
+our $row;
+sub fetch_row {
+       $row = $sth->fetchrow_hashref();
+       if ( ! $row && $sth->rows == $limit ) {
+               $offset += $limit;
+               select_table;
+               $row = $sth->fetchrow_hashref();
+       }
+       return $row;
+}
 
 our $join_offset = 0;
 our $sth_join;
@@ -66,21 +90,20 @@ sub join_table {
                from $table2
                where $fk > ?
                order by $fk asc
-               limit $join_limit
+               limit $limit
                offset $join_offset
        });
-       print STDERR "$join_offset";
+       print STDERR " $table2:$join_offset ";
        $sth_join->execute( $last_id );
-       warn "# $table2 columns ",dump( $sth_join->{NAME} );
-       print "join ",$sth_join->rows," from $table2 offset $join_offset limit $join_limit\n";
+#      warn "# $table2 columns ",dump( $sth_join->{NAME} ) if $join_offset = 0;
+       print STDERR " join ",$sth_join->rows, " rows ";
 }
 
 our $row_join;
-
 sub fetch_row_join {
        $row_join = $sth_join->fetchrow_hashref();
-       if ( ! $row_join && $sth_join->rows == $join_limit ) {
-               $join_offset += $join_limit;
+       if ( ! $row_join && $sth_join->rows == $limit ) {
+               $join_offset += $limit;
                join_table;
                $row_join = $sth_join->fetchrow_hashref();
        }
@@ -93,10 +116,11 @@ sub guess_types {
        return $row;
 }
 
+select_table;
 join_table;
 fetch_row_join;
 
-while (my $row = $sth->fetchrow_hashref() ) {
+while (my $row = fetch_row() ) {
 
        while ( $row_join && $row_join->{$fk} < $row->{$pk} ) {
                fetch_row_join;