fix pgpool down, check replication and add node

diff --git a/fix-pgpool.sh b/fix-pgpool.sh
new file mode 100755 (executable)
index 0000000..5b8679e
--- /dev/null
+++ b/fix-pgpool.sh
@@ -0,0 +1,17 @@
+#!/bin/sh -xe
+
+ssh postgres@10.200.1.60 PCPPASSFILE=/etc/pgpool-II/.pcppass pcp_watchdog_info --no-password | tee /dev/stderr | grep DEAD | cut -d' ' -f2 | while read host ; do
+       ssh -F ssh_config $host systemctl start pgpool
+done
+
+replication_ok=`psql -h 10.200.1.61 -t -A -c 'SELECT count(*) from pg_stat_replication' postgres postgres`
+if [ $replication_ok != 1 ] ; then
+       echo FIXME online-recovery?
+       exit 1
+fi
+
+psql -h 10.200.1.60 -p 9999 -c 'show pool_nodes' postgres postgres | grep down | awk '{ print $1 }' | while read node_id ; do
+       ssh postgres@10.200.1.60 PCPPASSFILE=/etc/pgpool-II/.pcppass pcp_attach_node --node-id $node_id --no-password
+done
+
+

diff --git a/ssh_config b/ssh_config
new file mode 100644 (file)
index 0000000..ea034cc
--- /dev/null
+++ b/ssh_config
@@ -0,0 +1,10 @@
+
+User root
+
+Host edozvola-db-pgpool
+       Hostname 10.200.1.60
+Host edozvola-db-01
+       Hostname 10.200.1.61
+Host edozvola-db-02
+       Hostname 10.200.1.62
+