3 * Copyright (C) Igor Sysoev
7 #include <ngx_config.h>
12 static ngx_int_t ngx_http_upstream_cmp_servers(const void *one,
15 ngx_http_upstream_get_peer(ngx_http_upstream_rr_peers_t *peers);
19 ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
20 ngx_http_upstream_srv_conf_t *us)
24 ngx_http_upstream_server_t *server;
25 ngx_http_upstream_rr_peers_t *peers, *backup;
27 us->peer.init = ngx_http_upstream_init_round_robin_peer;
30 server = us->servers->elts;
34 for (i = 0; i < us->servers->nelts; i++) {
35 if (server[i].backup) {
39 n += server[i].naddrs;
42 peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
43 + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
48 peers->single = (n == 1);
50 peers->name = &us->host;
54 for (i = 0; i < us->servers->nelts; i++) {
55 for (j = 0; j < server[i].naddrs; j++) {
56 if (server[i].backup) {
60 peers->peer[n].sockaddr = server[i].addrs[j].sockaddr;
61 peers->peer[n].socklen = server[i].addrs[j].socklen;
62 peers->peer[n].name = server[i].addrs[j].name;
63 peers->peer[n].max_fails = server[i].max_fails;
64 peers->peer[n].fail_timeout = server[i].fail_timeout;
65 peers->peer[n].down = server[i].down;
66 peers->peer[n].weight = server[i].down ? 0 : server[i].weight;
67 peers->peer[n].current_weight = peers->peer[n].weight;
72 us->peer.data = peers;
74 ngx_sort(&peers->peer[0], (size_t) n,
75 sizeof(ngx_http_upstream_rr_peer_t),
76 ngx_http_upstream_cmp_servers);
82 for (i = 0; i < us->servers->nelts; i++) {
83 if (!server[i].backup) {
87 n += server[i].naddrs;
94 backup = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
95 + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
103 backup->name = &us->host;
107 for (i = 0; i < us->servers->nelts; i++) {
108 for (j = 0; j < server[i].naddrs; j++) {
109 if (!server[i].backup) {
113 backup->peer[n].sockaddr = server[i].addrs[j].sockaddr;
114 backup->peer[n].socklen = server[i].addrs[j].socklen;
115 backup->peer[n].name = server[i].addrs[j].name;
116 backup->peer[n].weight = server[i].weight;
117 backup->peer[n].current_weight = server[i].weight;
118 backup->peer[n].max_fails = server[i].max_fails;
119 backup->peer[n].fail_timeout = server[i].fail_timeout;
120 backup->peer[n].down = server[i].down;
125 peers->next = backup;
127 ngx_sort(&backup->peer[0], (size_t) n,
128 sizeof(ngx_http_upstream_rr_peer_t),
129 ngx_http_upstream_cmp_servers);
135 /* an upstream implicitly defined by proxy_pass, etc. */
137 if (us->port == 0 && us->default_port == 0) {
138 ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
139 "no port in upstream \"%V\" in %s:%ui",
140 &us->host, us->file_name, us->line);
144 ngx_memzero(&u, sizeof(ngx_url_t));
147 u.port = (in_port_t) (us->port ? us->port : us->default_port);
149 if (ngx_inet_resolve_host(cf->pool, &u) != NGX_OK) {
151 ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
152 "%s in upstream \"%V\" in %s:%ui",
153 u.err, &us->host, us->file_name, us->line);
161 peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
162 + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
167 peers->single = (n == 1);
169 peers->name = &us->host;
171 for (i = 0; i < u.naddrs; i++) {
172 peers->peer[i].sockaddr = u.addrs[i].sockaddr;
173 peers->peer[i].socklen = u.addrs[i].socklen;
174 peers->peer[i].name = u.addrs[i].name;
175 peers->peer[i].weight = 1;
176 peers->peer[i].current_weight = 1;
177 peers->peer[i].max_fails = 1;
178 peers->peer[i].fail_timeout = 10;
181 us->peer.data = peers;
183 /* implicitly defined upstream has no backup servers */
190 ngx_http_upstream_cmp_servers(const void *one, const void *two)
192 ngx_http_upstream_rr_peer_t *first, *second;
194 first = (ngx_http_upstream_rr_peer_t *) one;
195 second = (ngx_http_upstream_rr_peer_t *) two;
197 return (first->weight < second->weight);
202 ngx_http_upstream_init_round_robin_peer(ngx_http_request_t *r,
203 ngx_http_upstream_srv_conf_t *us)
206 ngx_http_upstream_rr_peer_data_t *rrp;
208 rrp = r->upstream->peer.data;
211 rrp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_rr_peer_data_t));
216 r->upstream->peer.data = rrp;
219 rrp->peers = us->peer.data;
222 if (rrp->peers->number <= 8 * sizeof(uintptr_t)) {
223 rrp->tried = &rrp->data;
227 n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
228 / (8 * sizeof(uintptr_t));
230 rrp->tried = ngx_pcalloc(r->pool, n * sizeof(uintptr_t));
231 if (rrp->tried == NULL) {
236 r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
237 r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
238 r->upstream->peer.tries = rrp->peers->number;
240 r->upstream->peer.set_session =
241 ngx_http_upstream_set_round_robin_peer_session;
242 r->upstream->peer.save_session =
243 ngx_http_upstream_save_round_robin_peer_session;
251 ngx_http_upstream_create_round_robin_peer(ngx_http_request_t *r,
252 ngx_http_upstream_resolved_t *ur)
257 struct sockaddr_in *sin;
258 ngx_http_upstream_rr_peers_t *peers;
259 ngx_http_upstream_rr_peer_data_t *rrp;
261 rrp = r->upstream->peer.data;
264 rrp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_rr_peer_data_t));
269 r->upstream->peer.data = rrp;
272 peers = ngx_pcalloc(r->pool, sizeof(ngx_http_upstream_rr_peers_t)
273 + sizeof(ngx_http_upstream_rr_peer_t) * (ur->naddrs - 1));
278 peers->single = (ur->naddrs == 1);
279 peers->number = ur->naddrs;
280 peers->name = &ur->host;
283 peers->peer[0].sockaddr = ur->sockaddr;
284 peers->peer[0].socklen = ur->socklen;
285 peers->peer[0].name = ur->host;
286 peers->peer[0].weight = 1;
287 peers->peer[0].current_weight = 1;
288 peers->peer[0].max_fails = 1;
289 peers->peer[0].fail_timeout = 10;
293 for (i = 0; i < ur->naddrs; i++) {
295 len = NGX_INET_ADDRSTRLEN + sizeof(":65536") - 1;
297 p = ngx_pnalloc(r->pool, len);
302 len = ngx_inet_ntop(AF_INET, &ur->addrs[i], p, NGX_INET_ADDRSTRLEN);
303 len = ngx_sprintf(&p[len], ":%d", ur->port) - p;
305 sin = ngx_pcalloc(r->pool, sizeof(struct sockaddr_in));
310 sin->sin_family = AF_INET;
311 sin->sin_port = htons(ur->port);
312 sin->sin_addr.s_addr = ur->addrs[i];
314 peers->peer[i].sockaddr = (struct sockaddr *) sin;
315 peers->peer[i].socklen = sizeof(struct sockaddr_in);
316 peers->peer[i].name.len = len;
317 peers->peer[i].name.data = p;
318 peers->peer[i].weight = 1;
319 peers->peer[i].current_weight = 1;
320 peers->peer[i].max_fails = 1;
321 peers->peer[i].fail_timeout = 10;
328 if (rrp->peers->number <= 8 * sizeof(uintptr_t)) {
329 rrp->tried = &rrp->data;
333 n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
334 / (8 * sizeof(uintptr_t));
336 rrp->tried = ngx_pcalloc(r->pool, n * sizeof(uintptr_t));
337 if (rrp->tried == NULL) {
342 r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
343 r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
344 r->upstream->peer.tries = rrp->peers->number;
346 r->upstream->peer.set_session =
347 ngx_http_upstream_set_round_robin_peer_session;
348 r->upstream->peer.save_session =
349 ngx_http_upstream_save_round_robin_peer_session;
357 ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
359 ngx_http_upstream_rr_peer_data_t *rrp = data;
366 ngx_http_upstream_rr_peer_t *peer;
367 ngx_http_upstream_rr_peers_t *peers;
369 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
370 "get rr peer, try: %ui", pc->tries);
374 /* ngx_lock_mutex(rrp->peers->mutex); */
376 if (rrp->peers->last_cached) {
378 /* cached connection */
380 c = rrp->peers->cached[rrp->peers->last_cached];
381 rrp->peers->last_cached--;
383 /* ngx_unlock_mutex(ppr->peers->mutex); */
386 c->read->lock = c->read->own_lock;
387 c->write->lock = c->write->own_lock;
397 pc->connection = NULL;
399 if (rrp->peers->single) {
400 peer = &rrp->peers->peer[0];
404 /* there are several peers */
406 if (pc->tries == rrp->peers->number) {
408 /* it's a first try - get a current peer */
413 rrp->current = ngx_http_upstream_get_peer(rrp->peers);
415 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
416 "get rr peer, current: %ui %i",
418 rrp->peers->peer[rrp->current].current_weight);
420 n = rrp->current / (8 * sizeof(uintptr_t));
421 m = (uintptr_t) 1 << rrp->current % (8 * sizeof(uintptr_t));
423 if (!(rrp->tried[n] & m)) {
424 peer = &rrp->peers->peer[rrp->current];
428 if (peer->max_fails == 0
429 || peer->fails < peer->max_fails)
434 if (now - peer->accessed > peer->fail_timeout) {
439 peer->current_weight = 0;
448 if (pc->tries == 0) {
453 ngx_log_error(NGX_LOG_ALERT, pc->log, 0,
454 "round robin upstream stuck on %ui tries",
460 peer->current_weight--;
467 n = rrp->current / (8 * sizeof(uintptr_t));
468 m = (uintptr_t) 1 << rrp->current % (8 * sizeof(uintptr_t));
470 if (!(rrp->tried[n] & m)) {
472 peer = &rrp->peers->peer[rrp->current];
476 if (peer->max_fails == 0
477 || peer->fails < peer->max_fails)
482 if (now - peer->accessed > peer->fail_timeout) {
487 peer->current_weight = 0;
498 if (rrp->current >= rrp->peers->number) {
502 if (pc->tries == 0) {
507 ngx_log_error(NGX_LOG_ALERT, pc->log, 0,
508 "round robin upstream stuck on %ui tries",
514 peer->current_weight--;
520 pc->sockaddr = peer->sockaddr;
521 pc->socklen = peer->socklen;
522 pc->name = &peer->name;
524 /* ngx_unlock_mutex(rrp->peers->mutex); */
526 if (pc->tries == 1 && rrp->peers->next) {
527 pc->tries += rrp->peers->next->number;
529 n = rrp->peers->next->number / (8 * sizeof(uintptr_t)) + 1;
530 for (i = 0; i < n; i++) {
543 /* ngx_unlock_mutex(peers->mutex); */
545 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "backup servers");
547 rrp->peers = peers->next;
548 pc->tries = rrp->peers->number;
550 n = rrp->peers->number / (8 * sizeof(uintptr_t)) + 1;
551 for (i = 0; i < n; i++) {
555 rc = ngx_http_upstream_get_round_robin_peer(pc, rrp);
557 if (rc != NGX_BUSY) {
561 /* ngx_lock_mutex(peers->mutex); */
564 /* all peers failed, mark them as live for quick recovery */
566 for (i = 0; i < peers->number; i++) {
567 peers->peer[i].fails = 0;
570 /* ngx_unlock_mutex(peers->mutex); */
572 pc->name = peers->name;
579 ngx_http_upstream_get_peer(ngx_http_upstream_rr_peers_t *peers)
582 ngx_http_upstream_rr_peer_t *peer;
584 peer = &peers->peer[0];
588 for (i = 0; i < peers->number; i++) {
590 if (peer[i].current_weight <= 0) {
596 while (i < peers->number - 1) {
600 if (peer[i].current_weight <= 0) {
604 if (peer[n].current_weight * 1000 / peer[i].current_weight
605 > peer[n].weight * 1000 / peer[i].weight)
613 if (peer[i].current_weight > 0) {
620 for (i = 0; i < peers->number; i++) {
621 peer[i].current_weight = peer[i].weight;
628 ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
631 ngx_http_upstream_rr_peer_data_t *rrp = data;
634 ngx_http_upstream_rr_peer_t *peer;
636 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
637 "free rr peer %ui %ui", pc->tries, state);
639 if (state == 0 && pc->tries == 0) {
643 /* TODO: NGX_PEER_KEEPALIVE */
645 if (rrp->peers->single) {
650 if (state & NGX_PEER_FAILED) {
653 peer = &rrp->peers->peer[rrp->current];
655 /* ngx_lock_mutex(rrp->peers->mutex); */
658 peer->accessed = now;
660 if (peer->max_fails) {
661 peer->current_weight -= peer->weight / peer->max_fails;
664 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
665 "free rr peer failed: %ui %i",
666 rrp->current, peer->current_weight);
668 if (peer->current_weight < 0) {
669 peer->current_weight = 0;
672 /* ngx_unlock_mutex(rrp->peers->mutex); */
677 if (rrp->current >= rrp->peers->number) {
685 /* ngx_unlock_mutex(rrp->peers->mutex); */
692 ngx_http_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc,
695 ngx_http_upstream_rr_peer_data_t *rrp = data;
698 ngx_ssl_session_t *ssl_session;
699 ngx_http_upstream_rr_peer_t *peer;
701 peer = &rrp->peers->peer[rrp->current];
703 /* TODO: threads only mutex */
704 /* ngx_lock_mutex(rrp->peers->mutex); */
706 ssl_session = peer->ssl_session;
708 rc = ngx_ssl_set_session(pc->connection, ssl_session);
710 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
711 "set session: %p:%d",
712 ssl_session, ssl_session ? ssl_session->references : 0);
714 /* ngx_unlock_mutex(rrp->peers->mutex); */
721 ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc,
724 ngx_http_upstream_rr_peer_data_t *rrp = data;
726 ngx_ssl_session_t *old_ssl_session, *ssl_session;
727 ngx_http_upstream_rr_peer_t *peer;
729 ssl_session = ngx_ssl_get_session(pc->connection);
731 if (ssl_session == NULL) {
735 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
736 "save session: %p:%d", ssl_session, ssl_session->references);
738 peer = &rrp->peers->peer[rrp->current];
740 /* TODO: threads only mutex */
741 /* ngx_lock_mutex(rrp->peers->mutex); */
743 old_ssl_session = peer->ssl_session;
744 peer->ssl_session = ssl_session;
746 /* ngx_unlock_mutex(rrp->peers->mutex); */
748 if (old_ssl_session) {
750 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
751 "old session: %p:%d",
752 old_ssl_session, old_ssl_session->references);
754 /* TODO: may block */
756 ngx_ssl_free_session(old_ssl_session);