#!/usr/bin/perl -w
sub BEGIN {
-$ENV{DBI_AUTOPROXY}='dbi:Gofer:transport=stream;url=ssh:dpavlin@koha.ffzg.hr';
+#$ENV{DBI_AUTOPROXY}='dbi:Gofer:transport=stream;url=ssh:dpavlin@koha.ffzg.hr';
}
use strict;
use DBI;
use MongoDB;
use Data::Dump qw/dump/;
+use Getopt::Long;
-$|++;
+my $debug = 0;
+my $drop = 0;
+my $limit = 10000;
-my $debug = @ARGV ? 1 : 0;
+GetOptions(
+ 'debug!' => \$debug,
+ 'drop!' => \$drop,
+ 'limit=i' => \$limit,
+) || die $!;
our ( $dbi, $user, $password ) = ( "DBI:mysql:database=test" );
-our ( $database, $collection ) = ( 'test', 'test' );
+our ( $database, $collection ) = ( '', '' );
our ( $table, $pk ) = ( 'biblio' => 'biblionumber' );
our ( $table2, $fk ) = ( 'biblioitems' => 'biblionumber' );
-my $limit = 50000;
-my $join_limit = 10000;
-
-require 'config.pl';
+my $config = shift @ARGV || die "usage: $0 config.pl\n";
+require $config;
warn "# $dbi $user -> $database $collection $table.$pk<->$table2.$fk\n";
# mysql_enable_utf8 => 1,
});
-$db->drop if $debug;
+$db->drop if $drop;
# db.items.find().sort({_id:-1}).limit(1);
my $last = $coll->query()->sort({ '_id' => -1 })->limit(1)->next;
print "import $table.$pk > $last_id from $dbi\n";
-my $sth = $dbh->prepare(qq{
- select
- $pk as _id,
- $table.*
- from $table
- where $pk > ?
- order by $pk asc
- limit $limit
-});
+our $offset = 0;
+our $sth;
+
+sub select_table {
+
+ $sth = $dbh->prepare(qq{
+ select
+ $pk as _id,
+ $table.*
+ from $table
+ where $pk > ?
+ order by $pk asc
+ limit $limit
+ offset $offset
+ });
-$sth->execute( $last_id );
-warn "# $table columns ",dump( $sth->{NAME} );
-print "import ",$sth->rows," from $table\n";
+ print STDERR " $table:$offset ";
+ $sth->execute( $last_id );
+# warn "# $table columns ",dump( $sth->{NAME} ) if $offset == 0;
+ print STDERR " join ", $sth->rows, " rows ";
+}
+
+our $row;
+sub fetch_row {
+ $row = $sth->fetchrow_hashref();
+ if ( ! $row && $sth->rows == $limit ) {
+ $offset += $limit;
+ select_table;
+ $row = $sth->fetchrow_hashref();
+ }
+ return $row;
+}
our $join_offset = 0;
our $sth_join;
-our $join_more = 0;
sub join_table {
$sth_join = $dbh->prepare(qq{
from $table2
where $fk > ?
order by $fk asc
- limit $join_limit
+ limit $limit
offset $join_offset
});
- print STDERR "$join_offset";
+ print STDERR " $table2:$join_offset ";
$sth_join->execute( $last_id );
- warn "# $table2 columns ",dump( $sth_join->{NAME} );
- print "join ",$sth_join->rows," from $table2 offset $join_offset limit $join_limit\n";
- $join_more = $sth_join->rows == $join_limit ? 1 : 0;
+# warn "# $table2 columns ",dump( $sth_join->{NAME} ) if $join_offset = 0;
+ print STDERR " join ",$sth_join->rows, " rows ";
}
our $row_join;
-
sub fetch_row_join {
$row_join = $sth_join->fetchrow_hashref();
- if ( ! $row_join && $join_more ) {
- $join_offset += $join_limit;
+ if ( ! $row_join && $sth_join->rows == $limit ) {
+ $join_offset += $limit;
join_table;
$row_join = $sth_join->fetchrow_hashref();
}
return $row;
}
+select_table;
join_table;
fetch_row_join;
-while (my $row = $sth->fetchrow_hashref() ) {
+while (my $row = fetch_row() ) {
while ( $row_join && $row_join->{$fk} < $row->{$pk} ) {
fetch_row_join;