From: Dobrica Pavlinusic <dpavlin@rot13.org>
Date: Sun, 14 Nov 2010 12:30:04 +0000 (+0100)
Subject: import records in batches, support w=2
X-Git-Url: http://git.rot13.org/?a=commitdiff_plain;ds=sidebyside;h=9881f01109eb5547d6743c522c6d43fc2030c334;p=NoSQL-toys.git

import records in batches, support w=2
---

diff --git a/koha/RiakSearch.pm b/koha/RiakSearch.pm
index be2bcb5..b80c16c 100644
--- a/koha/RiakSearch.pm
+++ b/koha/RiakSearch.pm
@@ -19,6 +19,7 @@ sub new {
 	return bless {
 		ua  => $ua,
 		url => $url,
+		args => '',
 	}, $class;
 }
 
@@ -28,8 +29,10 @@ sub request {
 	my $full_url = $self->{url} . "/riak/$uri";
 	$full_url =~ s{//+}{/}g;
 	$full_url =~ s{http:/}{http://};
+	$full_url .= '?' . $self->{args} if $self->{args};
 
 	$headers->{'Content-Type'} = 'application/json' unless exists $headers->{'Content-Type'};
+	$headers->{'Host'} = $1 if $full_url =~ m{http://([^/]+)};
 
 	my $req;
 
@@ -41,7 +44,6 @@ sub request {
 	}
 
 	$req->header( $_ => $headers->{$_} ) foreach keys %$headers;
-	$req->header( 'Host' => $1 ) if $self->{url} =~ m{http://([^/]+)};
 
 #	warn "## $method $full_url ", dump($req, $content);
 	my $response = $self->{ua}->request($req);
diff --git a/koha/koha2riak-search.pl b/koha/koha2riak-search.pl
index 2fb5b4b..0d7df0b 100755
--- a/koha/koha2riak-search.pl
+++ b/koha/koha2riak-search.pl
@@ -6,35 +6,38 @@ use RiakSearch;
 use Data::Dump qw/dump/;
 
 
-my ( $riak_url, $dbi ) = @ARGV;
+my ( $riak_url, $table, $dbi ) = @ARGV;
 
 $riak_url ||= 'http://10.60.0.92:8098';
-$dbi ||= 'DBI:mysql:dbname=koha;host=10.60.0.10;port=3306';
-my @tables = qw(
-biblioitems
-biblio
-);
+$table    ||= 'biblioitems';
+$dbi      ||= 'DBI:mysql:dbname=koha;host=10.60.0.10;port=3306';
 
-my $limit;
-$limit = $ENV{LIMIT} if $ENV{LIMIT};
+my $batch_size = 1000;
 
 my $dbh = DBI->connect($dbi,"","") || die $DBI::errstr;
 my $riak = RiakSearch->new( $riak_url );
+#$riak->{args} = 'w=2';
 
-warn $riak->request( 'GET' => '/koha.marcxml' );
 
 sub riak_search_kv_hook {
-	$riak->request( 'PUT' => shift, { props => {
+	my $bucket = shift;
+	$riak->request( 'PUT' => $bucket, { props => {
 		precommit => [ { mod => 'riak_search_kv_hook', fun => 'precommit' } ],
 #		precommit => [],
 #		last_write_wins => 'true',
 		}
 	});
+	warn "riak_search_kv_hook $bucket ", $riak->request( 'GET' => '/koha.marcxml' );
 }
 
 riak_search_kv_hook 'koha.marcxml';
+riak_search_kv_hook "koha.$table";
 
-foreach my $table ( @tables ) {
+my $offset = 0;
+while(1) {
+
+	my $limit = "LIMIT $batch_size OFFSET $offset";
+	warn "SELECT * FROM $table $limit\n";
 
     my $sth = $dbh->prepare(qq{ select * from $table $limit}) || die $dbh->errstr();
     $sth->execute || die $sth->errstr();
@@ -42,8 +45,6 @@ foreach my $table ( @tables ) {
 
     print "import ", $sth->rows, " rows from $table pk:",dump( @pk ),"...\n";
 
-	riak_search_kv_hook "koha.$table";
-
     while (my $row = $sth->fetchrow_hashref() ) {
 
         my $key = join('_', map { $row->{$_} } @pk);
@@ -65,4 +66,9 @@ foreach my $table ( @tables ) {
 		$riak->request( 'PUT' => "/koha.$table/$key", $row, $headers );
     }
 
+	$offset += $sth->rows;
+	last if $sth->rows < $batch_size;
+
 }
+
+warn "END total_rows $offset\n";