From 97ef071d42e94afeb073bf77ea4c4707c7ae669f Mon Sep 17 00:00:00 2001 From: Dobrica Pavlinusic Date: Thu, 5 Feb 2009 22:26:53 +0100 Subject: [PATCH] nginx_upstream_hash-0.3.tar.gz http://wiki.codemongers.com/NginxHttpUpstreamRequestHashModule --- nginx_upstream_hash/CHANGES | 9 + nginx_upstream_hash/CREDITS | 4 + nginx_upstream_hash/README | 21 ++ nginx_upstream_hash/config | 3 + nginx_upstream_hash/nginx.patch | 14 + .../ngx_http_upstream_hash_module.c | 318 ++++++++++++++++++ nginx_upstream_hash/t/clean.sh | 5 + nginx_upstream_hash/t/hashtest.php | 61 ++++ nginx_upstream_hash/t/nginx.conf | 85 +++++ nginx_upstream_hash/t/nginx/conf/nginx.conf | 85 +++++ nginx_upstream_hash/t/restart.sh | 12 + 11 files changed, 617 insertions(+) create mode 100644 nginx_upstream_hash/CHANGES create mode 100644 nginx_upstream_hash/CREDITS create mode 100644 nginx_upstream_hash/README create mode 100644 nginx_upstream_hash/config create mode 100644 nginx_upstream_hash/nginx.patch create mode 100644 nginx_upstream_hash/ngx_http_upstream_hash_module.c create mode 100755 nginx_upstream_hash/t/clean.sh create mode 100644 nginx_upstream_hash/t/hashtest.php create mode 100644 nginx_upstream_hash/t/nginx.conf create mode 100644 nginx_upstream_hash/t/nginx/conf/nginx.conf create mode 100755 nginx_upstream_hash/t/restart.sh diff --git a/nginx_upstream_hash/CHANGES b/nginx_upstream_hash/CHANGES new file mode 100644 index 0000000..0d0670b --- /dev/null +++ b/nginx_upstream_hash/CHANGES @@ -0,0 +1,9 @@ + +Changes with upstream_hash 0.3 06 Aug 2008 + + *) Bugfix: infinite loop when retrying after a 404 and the "not_found" + flag of *_next_upstream was set. + + *) Change: no more "hash_method" directive. Hash method is always CRC-32. + + *) Change: failover strategy is compatible with PECL Memcache diff --git a/nginx_upstream_hash/CREDITS b/nginx_upstream_hash/CREDITS new file mode 100644 index 0000000..db980fc --- /dev/null +++ b/nginx_upstream_hash/CREDITS @@ -0,0 +1,4 @@ +All development (so far) by Evan Miller. + +PECL Memcache compatibility sponsored by Spil Games, Inc. +[http://www.spilgames.com/] diff --git a/nginx_upstream_hash/README b/nginx_upstream_hash/README new file mode 100644 index 0000000..c0dcbf1 --- /dev/null +++ b/nginx_upstream_hash/README @@ -0,0 +1,21 @@ +== ngx_http_upstream_hash_module == + +Installation: + + cd nginx-0.7.1 # or whatever + patch -p0 < /path/to/this/directory/nginx.patch + ./configure --add-module=/path/to/this/directory + make + make install + +Usage: + + upstream backend { + ... + hash $request_uri; + hash_again 10; # default 0 + } + +See http://wiki.codemongers.com/NginxHttpUpstreamHashModule for more details. + +Questions/patches to Evan Miller, emmiller@gmail.com. diff --git a/nginx_upstream_hash/config b/nginx_upstream_hash/config new file mode 100644 index 0000000..168bd20 --- /dev/null +++ b/nginx_upstream_hash/config @@ -0,0 +1,3 @@ +ngx_addon_name=ngx_http_upstream_hash_module +HTTP_MODULES="$HTTP_MODULES ngx_http_upstream_hash_module" +NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ngx_addon_dir/ngx_http_upstream_hash_module.c" diff --git a/nginx_upstream_hash/nginx.patch b/nginx_upstream_hash/nginx.patch new file mode 100644 index 0000000..8d41fa8 --- /dev/null +++ b/nginx_upstream_hash/nginx.patch @@ -0,0 +1,14 @@ +diff -Naur src/http/ngx_http_upstream.h src-hash/http/ngx_http_upstream.h +--- src/http/ngx_http_upstream.h 2008-03-03 12:04:06.000000000 -0800 ++++ src-hash/http/ngx_http_upstream.h 2008-06-07 13:01:37.000000000 -0700 +@@ -90,6 +90,10 @@ + + ngx_array_t *servers; /* ngx_http_upstream_server_t */ + ++ ngx_array_t *values; ++ ngx_array_t *lengths; ++ ngx_uint_t retries; ++ + ngx_uint_t flags; + ngx_str_t host; + u_char *file_name; diff --git a/nginx_upstream_hash/ngx_http_upstream_hash_module.c b/nginx_upstream_hash/ngx_http_upstream_hash_module.c new file mode 100644 index 0000000..9f0ab7a --- /dev/null +++ b/nginx_upstream_hash/ngx_http_upstream_hash_module.c @@ -0,0 +1,318 @@ +/* + * Hash a variable to choose an upstream server. + * + * Copyright (C) Evan Miller + * + * This module can be distributed under the same terms as Nginx itself. + */ + + +#include +#include +#include + +#define ngx_bitvector_index(index) index / (8 * sizeof(uintptr_t)) +#define ngx_bitvector_bit(index) (uintptr_t) 1 << index % (8 * sizeof(uintptr_t)) + +typedef struct { + struct sockaddr *sockaddr; + socklen_t socklen; + ngx_str_t name; +} ngx_http_upstream_hash_peer_t; + +typedef struct { + ngx_uint_t number; + ngx_http_upstream_hash_peer_t peer[0]; +} ngx_http_upstream_hash_peers_t; + +typedef struct { + ngx_http_upstream_hash_peers_t *peers; + ngx_uint_t hash; + ngx_str_t current_key; + ngx_str_t original_key; + ngx_uint_t try_i; + uintptr_t tried[1]; +} ngx_http_upstream_hash_peer_data_t; + + +static ngx_int_t ngx_http_upstream_init_hash_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us); +static ngx_int_t ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, + void *data); +static void ngx_http_upstream_free_hash_peer(ngx_peer_connection_t *pc, + void *data, ngx_uint_t state); +static char *ngx_http_upstream_hash(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); +static char *ngx_http_upstream_hash_again(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); +static ngx_int_t ngx_http_upstream_init_hash(ngx_conf_t *cf, + ngx_http_upstream_srv_conf_t *us); +static ngx_uint_t ngx_http_upstream_hash_crc32(u_char *keydata, size_t keylen); + + +static ngx_command_t ngx_http_upstream_hash_commands[] = { + { ngx_string("hash"), + NGX_HTTP_UPS_CONF|NGX_CONF_TAKE1, + ngx_http_upstream_hash, + 0, + 0, + NULL }, + + { ngx_string("hash_again"), + NGX_HTTP_UPS_CONF|NGX_CONF_TAKE1, + ngx_http_upstream_hash_again, + 0, + 0, + NULL }, + + ngx_null_command +}; + + +static ngx_http_module_t ngx_http_upstream_hash_module_ctx = { + NULL, /* preconfiguration */ + NULL, /* postconfiguration */ + + NULL, /* create main configuration */ + NULL, /* init main configuration */ + + NULL, /* create server configuration */ + NULL, /* merge server configuration */ + + NULL, /* create location configuration */ + NULL /* merge location configuration */ +}; + + +ngx_module_t ngx_http_upstream_hash_module = { + NGX_MODULE_V1, + &ngx_http_upstream_hash_module_ctx, /* module context */ + ngx_http_upstream_hash_commands, /* module directives */ + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +static ngx_int_t +ngx_http_upstream_init_hash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) +{ + ngx_uint_t i, j, n; + ngx_http_upstream_server_t *server; + ngx_http_upstream_hash_peers_t *peers; + + us->peer.init = ngx_http_upstream_init_hash_peer; + + if (!us->servers) { + + return NGX_ERROR; + } + + server = us->servers->elts; + + for (n = 0, i = 0; i < us->servers->nelts; i++) { + n += server[i].naddrs; + } + + peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_hash_peers_t) + + sizeof(ngx_http_upstream_hash_peer_t) * n); + + if (peers == NULL) { + return NGX_ERROR; + } + + peers->number = n; + + /* one hostname can have multiple IP addresses in DNS */ + for (n = 0, i = 0; i < us->servers->nelts; i++) { + for (j = 0; j < server[i].naddrs; j++, n++) { + peers->peer[n].sockaddr = server[i].addrs[j].sockaddr; + peers->peer[n].socklen = server[i].addrs[j].socklen; + peers->peer[n].name = server[i].addrs[j].name; + } + } + + us->peer.data = peers; + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_upstream_init_hash_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us) +{ + ngx_http_upstream_hash_peer_data_t *uhpd; + + ngx_str_t val; + + if (ngx_http_script_run(r, &val, us->lengths, 0, us->values) == NULL) { + return NGX_ERROR; + } + + uhpd = ngx_pcalloc(r->pool, sizeof(ngx_http_upstream_hash_peer_data_t) + + sizeof(uintptr_t) * + ((ngx_http_upstream_hash_peers_t *)us->peer.data)->number / + (8 * sizeof(uintptr_t))); + if (uhpd == NULL) { + return NGX_ERROR; + } + + r->upstream->peer.data = uhpd; + + uhpd->peers = us->peer.data; + + r->upstream->peer.free = ngx_http_upstream_free_hash_peer; + r->upstream->peer.get = ngx_http_upstream_get_hash_peer; + r->upstream->peer.tries = us->retries + 1; + + /* must be big enough for the retry keys */ + if ((uhpd->current_key.data = ngx_pcalloc(r->pool, NGX_ATOMIC_T_LEN + val.len)) == NULL) { + return NGX_ERROR; + } + + ngx_memcpy(uhpd->current_key.data, val.data, val.len); + uhpd->current_key.len = val.len; + uhpd->original_key = val; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "upstream_hash: hashing \"%V\"", &val); + + uhpd->hash = ngx_http_upstream_hash_crc32(val.data, val.len); + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data) +{ + ngx_http_upstream_hash_peer_data_t *uhpd = data; + ngx_http_upstream_hash_peer_t *peer; + ngx_uint_t peer_index; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "upstream_hash: get upstream request hash peer try %ui", pc->tries); + + pc->cached = 0; + pc->connection = NULL; + + peer_index = uhpd->hash % uhpd->peers->number; + + peer = &uhpd->peers->peer[peer_index]; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "upstream_hash: chose peer %ui w/ hash %ui", peer_index, uhpd->hash); + + pc->sockaddr = peer->sockaddr; + pc->socklen = peer->socklen; + pc->name = &peer->name; + + return NGX_OK; +} + +/* retry implementation is PECL memcache compatible */ +static void +ngx_http_upstream_free_hash_peer(ngx_peer_connection_t *pc, void *data, + ngx_uint_t state) +{ + ngx_http_upstream_hash_peer_data_t *uhpd = data; + ngx_uint_t current; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "upstream_hash: free upstream hash peer try %ui", pc->tries); + + if (state & (NGX_PEER_FAILED|NGX_PEER_NEXT) + && --pc->tries) + { + current = uhpd->hash % uhpd->peers->number; + + uhpd->tried[ngx_bitvector_index(current)] |= ngx_bitvector_bit(current); + + do { + uhpd->current_key.len = ngx_sprintf(uhpd->current_key.data, "%d%V", + ++uhpd->try_i, &uhpd->original_key) - uhpd->current_key.data; + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "upstream_hash: hashing \"%V\"", &uhpd->current_key); + uhpd->hash += ngx_http_upstream_hash_crc32(uhpd->current_key.data, + uhpd->current_key.len); + current = uhpd->hash % uhpd->peers->number; + } while ((uhpd->tried[ngx_bitvector_index(current)] & ngx_bitvector_bit(current)) + && --pc->tries); + } +} + +/* bit-shift, bit-mask, and non-zero requirement are for libmemcache compatibility */ +static ngx_uint_t +ngx_http_upstream_hash_crc32(u_char *keydata, size_t keylen) +{ + ngx_uint_t crc32 = (ngx_crc32_short(keydata, keylen) >> 16) & 0x7fff; + return crc32 ? crc32 : 1; +} + +static char * +ngx_http_upstream_hash(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_upstream_srv_conf_t *uscf; + ngx_http_script_compile_t sc; + ngx_str_t *value; + ngx_array_t *vars_lengths, *vars_values; + + value = cf->args->elts; + + ngx_memzero(&sc, sizeof(ngx_http_script_compile_t)); + + vars_lengths = NULL; + vars_values = NULL; + + sc.cf = cf; + sc.source = &value[1]; + sc.lengths = &vars_lengths; + sc.values = &vars_values; + sc.complete_lengths = 1; + sc.complete_values = 1; + + if (ngx_http_script_compile(&sc) != NGX_OK) { + return NGX_CONF_ERROR; + } + + uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); + + uscf->peer.init_upstream = ngx_http_upstream_init_hash; + + uscf->flags = NGX_HTTP_UPSTREAM_CREATE; + + uscf->values = vars_values->elts; + uscf->lengths = vars_lengths->elts; + + return NGX_CONF_OK; +} + +static char * +ngx_http_upstream_hash_again(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_upstream_srv_conf_t *uscf; + ngx_int_t n; + + ngx_str_t *value; + + uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); + + value = cf->args->elts; + + n = ngx_atoi(value[1].data, value[1].len); + + if (n == NGX_ERROR || n < 0) { + return "invalid number"; + } + + uscf->retries = n; + + return NGX_CONF_OK; +} diff --git a/nginx_upstream_hash/t/clean.sh b/nginx_upstream_hash/t/clean.sh new file mode 100755 index 0000000..5e74f4c --- /dev/null +++ b/nginx_upstream_hash/t/clean.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +rm -r nginx +mkdir -p nginx/conf/ +cp nginx.conf nginx/conf/ diff --git a/nginx_upstream_hash/t/hashtest.php b/nginx_upstream_hash/t/hashtest.php new file mode 100644 index 0000000..d798ca0 --- /dev/null +++ b/nginx_upstream_hash/t/hashtest.php @@ -0,0 +1,61 @@ +addServer("localhost", 11211 + $i); +} + +$test_vals = array( + 'quux' => 'baz', + 'foo' => 'bar', + 'shazam' => 'kerpow!', + 'verbum' => 'word', + 'felix' => 'happy', + 'ren' => 'stimpy', + 'Frank' => 'Julie', + 'peanuts' => 'cracker jacks', + 'all-gaul' => 'is divided into three parts', + 'the-more-tests-the-better' => 'i says', + 'adsfasw' => 'QA#(@()!@*$$*!!', + 'Swing-Low' => 'Sweet Cadillac', + 'can-has-exclamations!' => 'but no spaces or percents', + "Smile_if_you_like_UTF-8_\xa6\x3a" => "\xa6\x3b", + "8103*$)&^#@*^@!&!)*!_(#" => "whew" +); + +$curl = curl_init(); +curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1); + +foreach ($test_vals as $k => $v ) { + assert_equal($memcache->set("/$k", $v), TRUE, "Setting key \"$k\" via PECL"); + $memcache_url = rawurlencode($k); + curl_setopt($curl, CURLOPT_URL, "$base_url$memcache_url"); + assert_equal($v, curl_exec($curl), "Fetching key \"$k\" via Nginx"); +} + +curl_close($curl); + +if ($failures > 0) { + echo "$passes tests paseed, $failures tests failed\n"; +} else { + echo "All $passes tests passed\n"; +} diff --git a/nginx_upstream_hash/t/nginx.conf b/nginx_upstream_hash/t/nginx.conf new file mode 100644 index 0000000..dd1521e --- /dev/null +++ b/nginx_upstream_hash/t/nginx.conf @@ -0,0 +1,85 @@ + +#user nobody; +worker_processes 1; + +#error_log logs/error.log; +error_log logs/error.log debug; +#error_log logs/error.log notice; +#error_log logs/error.log info; + +#pid logs/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include mime.types; + default_type application/octet-stream; + + #log_format main '$remote_addr - $remote_user [$time_local] $request ' + # '"$status" $body_bytes_sent "$http_referer" ' + # '"$http_user_agent" "$http_x_forwarded_for"'; + + #access_log logs/access.log main; + + sendfile on; + #tcp_nopush on; + + #keepalive_timeout 0; + keepalive_timeout 65; + + #gzip on; + + upstream backend { +# active + server localhost:11211; + server localhost:11212; + server localhost:11213; + server localhost:11214; + server localhost:11215; + +# dead + server localhost:11216; + server localhost:11217; + server localhost:11218; + server localhost:11219; + server localhost:11220; + server localhost:11221; + server localhost:11222; + server localhost:11223; + server localhost:11224; + server localhost:11225; + server localhost:11226; + server localhost:11227; + server localhost:11228; + server localhost:11229; + server localhost:11230; + + hash $uri; + hash_again 64; + } + + server { + listen localhost:8081; + server_name localhost; + + #access_log logs/host.access.log main; + + location / { + set $memcached_key $uri; + memcached_pass backend; + memcached_next_upstream error timeout; + } + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root html; + } + + } +} diff --git a/nginx_upstream_hash/t/nginx/conf/nginx.conf b/nginx_upstream_hash/t/nginx/conf/nginx.conf new file mode 100644 index 0000000..dd1521e --- /dev/null +++ b/nginx_upstream_hash/t/nginx/conf/nginx.conf @@ -0,0 +1,85 @@ + +#user nobody; +worker_processes 1; + +#error_log logs/error.log; +error_log logs/error.log debug; +#error_log logs/error.log notice; +#error_log logs/error.log info; + +#pid logs/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include mime.types; + default_type application/octet-stream; + + #log_format main '$remote_addr - $remote_user [$time_local] $request ' + # '"$status" $body_bytes_sent "$http_referer" ' + # '"$http_user_agent" "$http_x_forwarded_for"'; + + #access_log logs/access.log main; + + sendfile on; + #tcp_nopush on; + + #keepalive_timeout 0; + keepalive_timeout 65; + + #gzip on; + + upstream backend { +# active + server localhost:11211; + server localhost:11212; + server localhost:11213; + server localhost:11214; + server localhost:11215; + +# dead + server localhost:11216; + server localhost:11217; + server localhost:11218; + server localhost:11219; + server localhost:11220; + server localhost:11221; + server localhost:11222; + server localhost:11223; + server localhost:11224; + server localhost:11225; + server localhost:11226; + server localhost:11227; + server localhost:11228; + server localhost:11229; + server localhost:11230; + + hash $uri; + hash_again 64; + } + + server { + listen localhost:8081; + server_name localhost; + + #access_log logs/host.access.log main; + + location / { + set $memcached_key $uri; + memcached_pass backend; + memcached_next_upstream error timeout; + } + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root html; + } + + } +} diff --git a/nginx_upstream_hash/t/restart.sh b/nginx_upstream_hash/t/restart.sh new file mode 100755 index 0000000..2256990 --- /dev/null +++ b/nginx_upstream_hash/t/restart.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +pkill nginx +cp nginx.conf ./nginx/conf/ +./nginx/sbin/nginx + +killall memcached +memcached -d -l 127.0.0.1 -p 11211 +memcached -d -l 127.0.0.1 -p 11212 +memcached -d -l 127.0.0.1 -p 11213 +memcached -d -l 127.0.0.1 -p 11214 +memcached -d -l 127.0.0.1 -p 11215 -- 2.20.1