diff options
| -rw-r--r-- | auto/modules | 14 | ||||
| -rw-r--r-- | auto/options | 3 | ||||
| -rw-r--r-- | src/event/ngx_event_connect.h | 7 | ||||
| -rw-r--r-- | src/http/modules/ngx_http_upstream_hash_module.c | 31 | ||||
| -rw-r--r-- | src/http/modules/ngx_http_upstream_ip_hash_module.c | 19 | ||||
| -rw-r--r-- | src/http/modules/ngx_http_upstream_least_conn_module.c | 16 | ||||
| -rw-r--r-- | src/http/modules/ngx_http_upstream_random_module.c | 38 | ||||
| -rw-r--r-- | src/http/modules/ngx_http_upstream_sticky_module.c | 592 | ||||
| -rw-r--r-- | src/http/modules/ngx_http_upstream_zone_module.c | 27 | ||||
| -rw-r--r-- | src/http/ngx_http_upstream.c | 30 | ||||
| -rw-r--r-- | src/http/ngx_http_upstream.h | 5 | ||||
| -rw-r--r-- | src/http/ngx_http_upstream_round_robin.c | 223 | ||||
| -rw-r--r-- | src/http/ngx_http_upstream_round_robin.h | 32 |
13 files changed, 1028 insertions, 9 deletions
diff --git a/auto/modules b/auto/modules index c199d89bf..f02691e16 100644 --- a/auto/modules +++ b/auto/modules @@ -950,6 +950,20 @@ if [ $HTTP = YES ]; then . auto/module fi + if [ $HTTP_UPSTREAM_STICKY = YES ]; then + have=NGX_HTTP_UPSTREAM_STICKY . auto/have + have=NGX_HTTP_UPSTREAM_SID . auto/have + + ngx_module_name=ngx_http_upstream_sticky_module + ngx_module_incs= + ngx_module_deps= + ngx_module_srcs=src/http/modules/ngx_http_upstream_sticky_module.c + ngx_module_libs= + ngx_module_link=$HTTP_UPSTREAM_STICKY + + . auto/module + fi + if [ $HTTP_STUB_STATUS = YES ]; then have=NGX_STAT_STUB . auto/have diff --git a/auto/options b/auto/options index 6a6e990a0..271153a76 100644 --- a/auto/options +++ b/auto/options @@ -107,6 +107,7 @@ HTTP_UPSTREAM_LEAST_CONN=YES HTTP_UPSTREAM_RANDOM=YES HTTP_UPSTREAM_KEEPALIVE=YES HTTP_UPSTREAM_ZONE=YES +HTTP_UPSTREAM_STICKY=YES # STUB HTTP_STUB_STATUS=NO @@ -292,6 +293,7 @@ $0: warning: the \"--with-ipv6\" option is deprecated" HTTP_UPSTREAM_RANDOM=NO ;; --without-http_upstream_keepalive_module) HTTP_UPSTREAM_KEEPALIVE=NO ;; --without-http_upstream_zone_module) HTTP_UPSTREAM_ZONE=NO ;; + --without-http_upstream_sticky) HTTP_UPSTREAM_STICKY=NO ;; --with-http_perl_module) HTTP_PERL=YES ;; --with-http_perl_module=dynamic) HTTP_PERL=DYNAMIC ;; @@ -516,6 +518,7 @@ cat << END disable ngx_http_upstream_keepalive_module --without-http_upstream_zone_module disable ngx_http_upstream_zone_module + --without-http_upstream_sticky disable sticky upstream modules --with-http_perl_module enable ngx_http_perl_module --with-http_perl_module=dynamic enable dynamic ngx_http_perl_module diff --git a/src/event/ngx_event_connect.h b/src/event/ngx_event_connect.h index d3b23782e..e428e7376 100644 --- a/src/event/ngx_event_connect.h +++ b/src/event/ngx_event_connect.h @@ -60,6 +60,11 @@ struct ngx_peer_connection_s { ngx_log_t *log; +#if (NGX_HTTP_UPSTREAM_SID || NGX_COMPAT) + ngx_str_t *hint; + ngx_str_t *sid; +#endif + unsigned cached:1; unsigned transparent:1; unsigned so_keepalive:1; @@ -68,7 +73,7 @@ struct ngx_peer_connection_s { /* ngx_connection_log_error_e */ unsigned log_error:2; - NGX_COMPAT_BEGIN(2) + NGX_COMPAT_BEGIN(1) NGX_COMPAT_END }; diff --git a/src/http/modules/ngx_http_upstream_hash_module.c b/src/http/modules/ngx_http_upstream_hash_module.c index 2ecc8d3fb..1835aee80 100644 --- a/src/http/modules/ngx_http_upstream_hash_module.c +++ b/src/http/modules/ngx_http_upstream_hash_module.c @@ -200,6 +200,17 @@ ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data) pc->cached = 0; pc->connection = NULL; +#if (NGX_HTTP_UPSTREAM_SID) + peer = ngx_http_upstream_get_rr_peer_by_sid(&hp->rrp, pc->hint, &p, 1); + + if (peer) { + n = p / (8 * sizeof(uintptr_t)); + m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); + + goto found; + } +#endif + for ( ;; ) { /* @@ -273,6 +284,10 @@ ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data) } } +#if (NGX_HTTP_UPSTREAM_SID) +found: +#endif + hp->rrp.current = peer; ngx_http_upstream_rr_peer_ref(hp->rrp.peers, peer); @@ -280,6 +295,10 @@ ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data) pc->socklen = peer->socklen; pc->name = &peer->name; +#if (NGX_HTTP_UPSTREAM_SID) + pc->sid = &peer->sid; +#endif + peer->conns++; if (now - peer->checked > peer->fail_timeout) { @@ -591,6 +610,14 @@ ngx_http_upstream_get_chash_peer(ngx_peer_connection_t *pc, void *data) points = hcf->points; point = &points->point[0]; +#if (NGX_HTTP_UPSTREAM_SID) + best = ngx_http_upstream_get_rr_peer_by_sid(&hp->rrp, pc->hint, &best_i, 0); + + if (best) { + goto found; + } +#endif + for ( ;; ) { server = point[hp->hash % points->number].server; @@ -671,6 +698,10 @@ found: pc->socklen = best->socklen; pc->name = &best->name; +#if (NGX_HTTP_UPSTREAM_SID) + pc->sid = &best->sid; +#endif + best->conns++; if (now - best->checked > best->fail_timeout) { diff --git a/src/http/modules/ngx_http_upstream_ip_hash_module.c b/src/http/modules/ngx_http_upstream_ip_hash_module.c index 1c1b41d19..8594698ed 100644 --- a/src/http/modules/ngx_http_upstream_ip_hash_module.c +++ b/src/http/modules/ngx_http_upstream_ip_hash_module.c @@ -183,6 +183,17 @@ ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data) hash = iphp->hash; +#if (NGX_HTTP_UPSTREAM_SID) + peer = ngx_http_upstream_get_rr_peer_by_sid(&iphp->rrp, pc->hint, &p, 1); + + if (peer) { + n = p / (8 * sizeof(uintptr_t)); + m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); + + goto found; + } +#endif + for ( ;; ) { for (i = 0; i < (ngx_uint_t) iphp->addrlen; i++) { @@ -239,6 +250,10 @@ ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data) } } +#if (NGX_HTTP_UPSTREAM_SID) +found: +#endif + iphp->rrp.current = peer; ngx_http_upstream_rr_peer_ref(iphp->rrp.peers, peer); @@ -246,6 +261,10 @@ ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data) pc->socklen = peer->socklen; pc->name = &peer->name; +#if (NGX_HTTP_UPSTREAM_SID) + pc->sid = &peer->sid; +#endif + peer->conns++; if (now - peer->checked > peer->fail_timeout) { diff --git a/src/http/modules/ngx_http_upstream_least_conn_module.c b/src/http/modules/ngx_http_upstream_least_conn_module.c index 4df97777c..066f457d1 100644 --- a/src/http/modules/ngx_http_upstream_least_conn_module.c +++ b/src/http/modules/ngx_http_upstream_least_conn_module.c @@ -138,6 +138,14 @@ ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) p = 0; #endif +#if (NGX_HTTP_UPSTREAM_SID) + best = ngx_http_upstream_get_rr_peer_by_sid(rrp, pc->hint, &p, 0); + + if (best) { + goto best_chosen; + } +#endif + for (peer = peers->peer, i = 0; peer; peer = peer->next, i++) @@ -239,6 +247,10 @@ ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) best->current_weight -= total; +#if (NGX_HTTP_UPSTREAM_SID) +best_chosen: +#endif + if (now - best->checked > best->fail_timeout) { best->checked = now; } @@ -247,6 +259,10 @@ ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) pc->socklen = best->socklen; pc->name = &best->name; +#if (NGX_HTTP_UPSTREAM_SID) + pc->sid = &best->sid; +#endif + best->conns++; rrp->current = best; diff --git a/src/http/modules/ngx_http_upstream_random_module.c b/src/http/modules/ngx_http_upstream_random_module.c index 74714874b..72885cde8 100644 --- a/src/http/modules/ngx_http_upstream_random_module.c +++ b/src/http/modules/ngx_http_upstream_random_module.c @@ -249,6 +249,17 @@ ngx_http_upstream_get_random_peer(ngx_peer_connection_t *pc, void *data) now = ngx_time(); +#if (NGX_HTTP_UPSTREAM_SID) + peer = ngx_http_upstream_get_rr_peer_by_sid(rrp, pc->hint, &i, 1); + + if (peer) { + n = i / (8 * sizeof(uintptr_t)); + m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); + + goto found; + } +#endif + for ( ;; ) { i = ngx_http_upstream_peek_random_peer(peers, rp); @@ -292,6 +303,10 @@ ngx_http_upstream_get_random_peer(ngx_peer_connection_t *pc, void *data) } } +#if (NGX_HTTP_UPSTREAM_SID) +found: +#endif + rrp->current = peer; ngx_http_upstream_rr_peer_ref(peers, peer); @@ -303,6 +318,10 @@ ngx_http_upstream_get_random_peer(ngx_peer_connection_t *pc, void *data) pc->socklen = peer->socklen; pc->name = &peer->name; +#if (NGX_HTTP_UPSTREAM_SID) + pc->sid = &peer->sid; +#endif + peer->conns++; ngx_http_upstream_rr_peer_unlock(peers, peer); @@ -357,6 +376,17 @@ ngx_http_upstream_get_random2_peer(ngx_peer_connection_t *pc, void *data) p = 0; #endif +#if (NGX_HTTP_UPSTREAM_SID) + peer = ngx_http_upstream_get_rr_peer_by_sid(rrp, pc->hint, &i, 0); + + if (peer) { + n = i / (8 * sizeof(uintptr_t)); + m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); + + goto found; + } +#endif + for ( ;; ) { i = ngx_http_upstream_peek_random_peer(peers, rp); @@ -410,6 +440,10 @@ ngx_http_upstream_get_random2_peer(ngx_peer_connection_t *pc, void *data) } } +#if (NGX_HTTP_UPSTREAM_SID) +found: +#endif + rrp->current = peer; ngx_http_upstream_rr_peer_ref(peers, peer); @@ -421,6 +455,10 @@ ngx_http_upstream_get_random2_peer(ngx_peer_connection_t *pc, void *data) pc->socklen = peer->socklen; pc->name = &peer->name; +#if (NGX_HTTP_UPSTREAM_SID) + pc->sid = &peer->sid; +#endif + peer->conns++; ngx_http_upstream_rr_peers_unlock(peers); diff --git a/src/http/modules/ngx_http_upstream_sticky_module.c b/src/http/modules/ngx_http_upstream_sticky_module.c new file mode 100644 index 000000000..fa0d14816 --- /dev/null +++ b/src/http/modules/ngx_http_upstream_sticky_module.c @@ -0,0 +1,592 @@ + +/* + * Copyright (C) Nginx, Inc. + */ + + +#include <ngx_config.h> +#include <ngx_core.h> +#include <ngx_http.h> +#include <ngx_md5.h> + + +#define NGX_HTTP_STICKY_COOKIE_MAX_EXPIRES 2145916555 + + +/* per-upstream sticky configuration */ +typedef struct { + ngx_http_upstream_init_pt original_init_upstream; + ngx_http_upstream_init_peer_pt original_init_peer; + + ngx_array_t *lookup_vars; /* of ngx_int_t */ + + ngx_str_t cookie_name; + ngx_str_t cookie_domain; + ngx_str_t cookie_path; + time_t cookie_expires; +} ngx_http_upstream_sticky_srv_conf_t; + + +typedef struct { + void *original_data; + ngx_http_request_t *request; + + ngx_http_upstream_sticky_srv_conf_t *conf; + + ngx_str_t id; + ngx_table_elt_t *cookie; + + ngx_event_get_peer_pt original_get_peer; + ngx_event_free_peer_pt original_free_peer; + +#if (NGX_HTTP_SSL) + ngx_event_set_peer_session_pt original_set_session; + ngx_event_save_peer_session_pt original_save_session; +#endif +} ngx_http_upstream_sticky_peer_data_t; + + +static ngx_int_t ngx_http_upstream_sticky_init_upstream(ngx_conf_t *cf, + ngx_http_upstream_srv_conf_t *us); +static ngx_int_t ngx_http_upstream_sticky_init_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us); +static ngx_int_t ngx_http_upstream_sticky_get_id( + ngx_http_upstream_sticky_srv_conf_t *stcf, ngx_http_request_t *r, + ngx_str_t *id); +static ngx_int_t ngx_http_upstream_sticky_get_peer(ngx_peer_connection_t *pc, + void *data); +static void ngx_http_upstream_sticky_free_peer(ngx_peer_connection_t *pc, + void *data, ngx_uint_t state); + + +#if (NGX_HTTP_SSL) +static ngx_int_t ngx_http_upstream_sticky_set_session( + ngx_peer_connection_t *pc, void *data); +static void ngx_http_upstream_sticky_save_session(ngx_peer_connection_t *pc, + void *data); +#endif + + +static ngx_int_t ngx_http_upstream_sticky_cookie_insert( + ngx_peer_connection_t *pc, ngx_http_upstream_sticky_peer_data_t *stp); + + +static void *ngx_http_upstream_sticky_create_conf(ngx_conf_t *cf); +static char *ngx_http_upstream_sticky(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); + + +static u_char expires[] = "; expires=Thu, 31-Dec-37 23:55:55 GMT"; + + +static ngx_command_t ngx_http_upstream_sticky_commands[] = { + + { ngx_string("sticky"), + NGX_HTTP_UPS_CONF|NGX_CONF_2MORE, + ngx_http_upstream_sticky, + 0, + 0, + NULL }, + + ngx_null_command +}; + + +static ngx_http_module_t +ngx_http_upstream_sticky_module_ctx = { + NULL, /* preconfiguration */ + NULL, /* postconfiguration */ + + NULL, /* create main configuration */ + NULL, /* init main configuration */ + + ngx_http_upstream_sticky_create_conf, /* create server configuration */ + NULL, /* merge server configuration */ + + NULL, /* create location configuration */ + NULL /* merge location configuration */ +}; + + +ngx_module_t +ngx_http_upstream_sticky_module = +{ + NGX_MODULE_V1, + + &ngx_http_upstream_sticky_module_ctx, /* module context */ + ngx_http_upstream_sticky_commands, /* module directives */ + + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + + NULL, /* init module */ + NULL, /* init process */ + + NULL, /* init thread */ + NULL, /* exit thread */ + + NULL, /* exit process */ + NULL, /* exit master */ + + NGX_MODULE_V1_PADDING +}; + + +static ngx_int_t +ngx_http_upstream_sticky_init_upstream(ngx_conf_t *cf, + ngx_http_upstream_srv_conf_t *us) +{ + ngx_http_upstream_sticky_srv_conf_t *stcf; + + stcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_sticky_module); + + if (stcf->original_init_upstream(cf, us) != NGX_OK) { + return NGX_ERROR; + } + + stcf->original_init_peer = us->peer.init; + us->peer.init = ngx_http_upstream_sticky_init_peer; + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_upstream_sticky_init_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us) +{ + ngx_int_t rc; + ngx_http_upstream_t *u; + ngx_http_upstream_sticky_srv_conf_t *stcf; + ngx_http_upstream_sticky_peer_data_t *stp; + + stcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_sticky_module); + + stp = ngx_pcalloc(r->pool, sizeof(ngx_http_upstream_sticky_peer_data_t)); + if (stp == NULL) { + return NGX_ERROR; + } + + rc = stcf->original_init_peer(r, us); + + if (rc != NGX_OK) { + return rc; + } + + u = r->upstream; + + stp->original_data = u->peer.data; + stp->original_get_peer = u->peer.get; + stp->original_free_peer = u->peer.free; + + stp->request = r; + stp->conf = stcf; + + u->peer.get = ngx_http_upstream_sticky_get_peer; + u->peer.free = ngx_http_upstream_sticky_free_peer; + u->peer.data = stp; + +#if (NGX_HTTP_SSL) + stp->original_set_session = u->peer.set_session; + stp->original_save_session = u->peer.save_session; + u->peer.set_session = ngx_http_upstream_sticky_set_session; + u->peer.save_session = ngx_http_upstream_sticky_save_session; +#endif + + ngx_http_upstream_sticky_get_id(stcf, r, &stp->id); + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_upstream_sticky_get_id(ngx_http_upstream_sticky_srv_conf_t *stcf, + ngx_http_request_t *r, ngx_str_t *id) +{ + ngx_int_t *index; + ngx_uint_t i; + ngx_http_variable_value_t *v; + + index = stcf->lookup_vars->elts; + + for (i = 0; i < stcf->lookup_vars->nelts; i++) { + + v = ngx_http_get_flushed_variable(r, index[i]); + + if (v == NULL || v->not_found || v->len == 0) { + continue; + } + + id->data = v->data; + id->len = v->len; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "sticky: using \"%v\" found in variable #%i", v, i + 1); + + return NGX_OK; + } + + ngx_str_null(id); + + return NGX_DONE; +} + + +static ngx_int_t +ngx_http_upstream_sticky_get_peer(ngx_peer_connection_t *pc, void *data) +{ + ngx_http_upstream_sticky_peer_data_t *stp = data; + + ngx_int_t rc; + + if (pc->hint == NULL && stp->id.len) { + pc->hint = &stp->id; + } + + rc = stp->original_get_peer(pc, stp->original_data); + + pc->hint = NULL; + + if (rc != NGX_OK && rc != NGX_DONE) { + return rc; + } + + if (stp->conf->cookie_name.len == 0) { + return rc; + } + + if (ngx_http_upstream_sticky_cookie_insert(pc, stp) != NGX_OK) { + return NGX_ERROR; + } + + return rc; +} + + +static void +ngx_http_upstream_sticky_free_peer(ngx_peer_connection_t *pc, void *data, + ngx_uint_t state) +{ + ngx_http_upstream_sticky_peer_data_t *stp = data; + + stp->original_free_peer(pc, stp->original_data, state); +} + + +#if (NGX_HTTP_SSL) + +static ngx_int_t +ngx_http_upstream_sticky_set_session(ngx_peer_connection_t *pc, void *data) +{ + ngx_http_upstream_sticky_peer_data_t *stp = data; + + return stp->original_set_session(pc, stp->original_data); +} + + +static void +ngx_http_upstream_sticky_save_session(ngx_peer_connection_t *pc, void *data) +{ + ngx_http_upstream_sticky_peer_data_t *stp = data; + + stp->original_save_session(pc, stp->original_data); +} + +#endif + + +static ngx_int_t +ngx_http_upstream_sticky_cookie_insert(ngx_peer_connection_t *pc, + ngx_http_upstream_sticky_peer_data_t *stp) +{ + size_t len; + u_char *data, *p; + ngx_table_elt_t *cookie; + ngx_http_request_t *r; + ngx_http_upstream_sticky_srv_conf_t *stcf; + + stcf = stp->conf; + r = stp->request; + + if (pc->sid == NULL) { + ngx_log_error(NGX_LOG_WARN, pc->log, 0, + "balancer does not support sticky"); + return NGX_OK; + } + +#if (NGX_DEBUG) + + if (stp->id.len) { + + /* check that the selected peer matches SID from request */ + + if (pc->sid->len != stp->id.len + || ngx_memcmp(pc->sid->data, stp->id.data, stp->id.len) != 0) + { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "sticky: server with requested SID is unavailable"); + } + } + +#endif + + len = stcf->cookie_name.len + 1 + pc->sid->len + stcf->cookie_domain.len + + stcf->cookie_path.len; + + if (stcf->cookie_expires != (time_t) NGX_CONF_UNSET) { + len += sizeof(expires) - 1; + } + + data = ngx_pnalloc(r->pool, len); + if (data == NULL) { + return NGX_ERROR; + } + + p = ngx_copy(data, stcf->cookie_name.data, stcf->cookie_name.len); + *p++ = '='; + p = ngx_copy(p, pc->sid->data, pc->sid->len); + + if (stcf->cookie_expires != (time_t) NGX_CONF_UNSET) { + + if (stcf->cookie_expires == NGX_HTTP_STICKY_COOKIE_MAX_EXPIRES) { + p = ngx_cpymem(p, expires, sizeof(expires) - 1); + + } else { + p = ngx_cpymem(p, "; expires=", 10); + p = ngx_http_cookie_time(p, ngx_time() + stcf->cookie_expires); + } + } + + p = ngx_copy(p, stcf->cookie_domain.data, stcf->cookie_domain.len); + ngx_memcpy(p, stcf->cookie_path.data, stcf->cookie_path.len); + + cookie = stp->cookie; + + if (cookie == NULL) { + + cookie = ngx_list_push(&r->headers_out.headers); + if (cookie == NULL) { + return NGX_ERROR; + } + + cookie->hash = 1; + cookie->next = NULL; + ngx_str_set(&cookie->key, "Set-Cookie"); + + stp->cookie = cookie; + } + + cookie->value.len = len; + cookie->value.data = data; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "sticky: set cookie: \"%V\"", &cookie->value); + + return NGX_OK; +} + + +static void * +ngx_http_upstream_sticky_create_conf(ngx_conf_t *cf) +{ + ngx_http_upstream_sticky_srv_conf_t *stcf; + + stcf = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_sticky_srv_conf_t)); + if (stcf == NULL) { + return NULL; + } + + /* + * set by ngx_pcalloc(): + * + * stcf->original_init_upstream = NULL; + * stcf->original_init_peer = NULL; + * + * stcf->lookup_vars = NULL; + * + * stcf->cookie_name = { 0, NULL }; + * stcf->cookie_domain = { 0, NULL }; + * stcf->cookie_path = { 0, NULL }; + */ + + stcf->cookie_expires = NGX_CONF_UNSET; + + return stcf; +} + + +static char * +ngx_http_upstream_sticky(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + u_char *p; + ngx_str_t *value, varname; + ngx_int_t index, *indexp; + ngx_uint_t i; + ngx_http_upstream_srv_conf_t *us; + ngx_http_upstream_sticky_srv_conf_t *stcf; + + us = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); + stcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_sticky_module); + + if (stcf->lookup_vars != NULL) { + return "is duplicate"; + } + + stcf->lookup_vars = ngx_array_create(cf->pool, 1, sizeof(ngx_int_t)); + if (stcf->lookup_vars == NULL) { + return NGX_CONF_ERROR; + } + + value = cf->args->elts; + + if (ngx_strcmp(value[1].data, "route") == 0) { + + for (i = 2; i < cf->args->nelts; i++) { + + if (value[i].data[0] != '$') { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid variable name \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + value[i].len--; + value[i].data++; + + index = ngx_http_get_variable_index(cf, &value[i]); + if (index == NGX_ERROR) { + return NGX_CONF_ERROR; + } + + indexp = ngx_array_push(stcf->lookup_vars); + if (indexp == NULL) { + return NGX_CONF_ERROR; + } + + *indexp = index; + } + + /* + * stcf->stick = NULL; + */ + + } else if (ngx_strcmp(value[1].data, "cookie") == 0) { + + if (value[2].len == 0) { + return "empty cookie name"; + } + + stcf->cookie_name = value[2]; + + for (i = 3; i < cf->args->nelts; i++) { + + if (ngx_strncmp(value[i].data, "domain=", 7) == 0) { + + if (stcf->cookie_domain.data != NULL) { + return "parameter \"domain\" is duplicate"; + } + + value[i].data += 7; + value[i].len -= 7; + + if (value[i].len == 0) { + return "no value for \"domain\""; + } + + stcf->cookie_domain.len = sizeof("; domain=") - 1 + + value[i].len; + + stcf->cookie_domain.data = ngx_pnalloc(cf->pool, + stcf->cookie_domain.len); + if (stcf->cookie_domain.data == NULL) { + return NGX_CONF_ERROR; + } + + p = ngx_cpymem(stcf->cookie_domain.data, + "; domain=", sizeof("; domain=") - 1); + ngx_memcpy(p, value[i].data, value[i].len); + + + } else if (ngx_strncmp(value[i].data, "path=", 5) == 0) { + + if (stcf->cookie_path.data != NULL) { + return "parameter \"path\" is duplicate"; + } + + value[i].data += 5; + value[i].len -= 5; + + if (value[i].len == 0) { + return "no value for \"path\""; + } + + stcf->cookie_path.len = sizeof("; path=") - 1 + value[i].len; + + stcf->cookie_path.data = ngx_pnalloc(cf->pool, + stcf->cookie_path.len); + if (stcf->cookie_path.data == NULL) { + return NGX_CONF_ERROR; + } + + p = ngx_cpymem(stcf->cookie_path.data, + "; path=", sizeof("; path=") - 1); + ngx_memcpy(p, value[i].data, value[i].len); + + + } else if (ngx_strncmp(value[i].data, "expires=", 8) == 0) { + + if (stcf->cookie_expires != (time_t) NGX_CONF_UNSET) { + return "parameter \"expires\" is duplicate"; + } + + value[i].data += 8; + value[i].len -= 8; + + if (ngx_strcmp(value[i].data, "max") == 0) { + stcf->cookie_expires = NGX_HTTP_STICKY_COOKIE_MAX_EXPIRES; + + } else { + stcf->cookie_expires = ngx_parse_time(&value[i], 1); + if (stcf->cookie_expires == (time_t) NGX_ERROR) { + return "invalid \"expires\" parameter value"; + } + } + + } else { + return "unknown parameter"; + } + } + + varname.len = sizeof("cookie_") - 1 + stcf->cookie_name.len; + varname.data = ngx_pnalloc(cf->pool, varname.len); + if (varname.data == NULL) { + return NGX_CONF_ERROR; + } + + ngx_sprintf(varname.data, "cookie_%V", &stcf->cookie_name); + + index = ngx_http_get_variable_index(cf, &varname); + if (index == NGX_ERROR) { + return NGX_CONF_ERROR; + } + + indexp = ngx_array_push(stcf->lookup_vars); + if (indexp == NULL) { + return NGX_CONF_ERROR; + } + + *indexp = index; + + } else { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "unknown parameter \"%V\"", + &value[1]); + return NGX_CONF_ERROR; + } + + stcf->original_init_upstream = us->peer.init_upstream + ? us->peer.init_upstream + : ngx_http_upstream_init_round_robin; + + us->peer.init_upstream = ngx_http_upstream_sticky_init_upstream; + + return NGX_CONF_OK; +} diff --git a/src/http/modules/ngx_http_upstream_zone_module.c b/src/http/modules/ngx_http_upstream_zone_module.c index 2ce8a7b5b..578b41d45 100644 --- a/src/http/modules/ngx_http_upstream_zone_module.c +++ b/src/http/modules/ngx_http_upstream_zone_module.c @@ -385,6 +385,9 @@ ngx_http_upstream_zone_copy_peer(ngx_http_upstream_rr_peers_t *peers, ngx_memcpy(dst, src, sizeof(ngx_http_upstream_rr_peer_t)); dst->sockaddr = NULL; dst->name.data = NULL; +#if (NGX_HTTP_UPSTREAM_SID) + dst->sid.data = NULL; +#endif dst->server.data = NULL; dst->host = NULL; } @@ -399,9 +402,19 @@ ngx_http_upstream_zone_copy_peer(ngx_http_upstream_rr_peers_t *peers, goto failed; } +#if (NGX_HTTP_UPSTREAM_SID) + dst->sid.data = ngx_slab_calloc_locked(pool, NGX_HTTP_UPSTREAM_SID_LEN); + if (dst->sid.data == NULL) { + goto failed; + } +#endif + if (src) { ngx_memcpy(dst->sockaddr, src->sockaddr, src->socklen); ngx_memcpy(dst->name.data, src->name.data, src->name.len); +#if (NGX_HTTP_UPSTREAM_SID) + ngx_memcpy(dst->sid.data, src->sid.data, src->sid.len); +#endif dst->server.data = ngx_slab_alloc_locked(pool, src->server.len); if (dst->server.data == NULL) { @@ -460,6 +473,12 @@ failed: ngx_slab_free_locked(pool, dst->server.data); } +#if (NGX_HTTP_UPSTREAM_SID) + if (dst->sid.data) { + ngx_slab_free_locked(pool, dst->sid.data); + } +#endif + if (dst->name.data) { ngx_slab_free_locked(pool, dst->name.data); } @@ -573,6 +592,10 @@ ngx_http_upstream_zone_preresolve(ngx_http_upstream_rr_peer_t *resolve, peer->fail_timeout = template->fail_timeout; peer->down = template->down; +#if (NGX_HTTP_UPSTREAM_SID) + ngx_http_upstream_copy_round_robin_sid(peer, template); +#endif + (*peers->config)++; *peerp = peer; @@ -966,6 +989,10 @@ again: peer->fail_timeout = template->fail_timeout; peer->down = template->down; +#if (NGX_HTTP_UPSTREAM_SID) + ngx_http_upstream_copy_round_robin_sid(peer, template); +#endif + *peerp = peer; peerp = &peer->next; diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index 74042b5ec..3ee3120c8 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -4593,6 +4593,10 @@ ngx_http_upstream_next(ngx_http_request_t *r, ngx_http_upstream_t *u, u->peer.free(&u->peer, u->peer.data, state); u->peer.sockaddr = NULL; + +#if (NGX_HTTP_UPSTREAM_SID) + u->peer.sid = NULL; +#endif } if (ft_type == NGX_HTTP_UPSTREAM_FT_TIMEOUT) { @@ -4776,6 +4780,10 @@ ngx_http_upstream_finalize_request(ngx_http_request_t *r, if (u->peer.free && u->peer.sockaddr) { u->peer.free(&u->peer, u->peer.data, 0); u->peer.sockaddr = NULL; + +#if (NGX_HTTP_UPSTREAM_SID) + u->peer.sid = NULL; +#endif } if (u->peer.connection) { @@ -6527,6 +6535,28 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) continue; } +#if (NGX_HTTP_UPSTREAM_SID) + if (ngx_strncmp(value[i].data, "route=", 6) == 0) { + + us->sid.data = &value[i].data[6]; + us->sid.len = value[i].len - 6; + + if (us->sid.len == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "route is empty"); + return NGX_CONF_ERROR; + } + + if (us->sid.len > NGX_HTTP_UPSTREAM_SID_LEN) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "route is longer than %d", + NGX_HTTP_UPSTREAM_SID_LEN); + return NGX_CONF_ERROR; + } + + continue; + } +#endif + #if (NGX_HTTP_UPSTREAM_ZONE) if (ngx_strcmp(value[i].data, "resolve") == 0) { resolve = 1; diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h index 6176e17b6..4f1663225 100644 --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -110,8 +110,9 @@ typedef struct { ngx_str_t service; #endif - NGX_COMPAT_BEGIN(2) - NGX_COMPAT_END +#if (NGX_HTTP_UPSTREAM_SID || NGX_COMPAT) + ngx_str_t sid; +#endif } ngx_http_upstream_server_t; diff --git a/src/http/ngx_http_upstream_round_robin.c b/src/http/ngx_http_upstream_round_robin.c index 4637318d1..afbb1f23e 100644 --- a/src/http/ngx_http_upstream_round_robin.c +++ b/src/http/ngx_http_upstream_round_robin.c @@ -8,14 +8,20 @@ #include <ngx_config.h> #include <ngx_core.h> #include <ngx_http.h> +#include <ngx_md5.h> #define ngx_http_upstream_tries(p) ((p)->tries \ + ((p)->next ? (p)->next->tries : 0)) +#if (NGX_HTTP_UPSTREAM_SID) +static ngx_int_t ngx_http_upstream_create_sid(ngx_conf_t *cf, + ngx_http_upstream_rr_peer_t *peer, ngx_str_t *route); +#endif + static ngx_http_upstream_rr_peer_t *ngx_http_upstream_get_peer( - ngx_http_upstream_rr_peer_data_t *rrp); + ngx_http_upstream_rr_peer_data_t *rrp, ngx_peer_connection_t *pc); #if (NGX_HTTP_SSL) @@ -190,6 +196,14 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, peer[n].down = server[i].down; peer[n].server = server[i].name; +#if (NGX_HTTP_UPSTREAM_SID) + if (ngx_http_upstream_create_sid(cf, &peer[n], &server[i].sid) + != NGX_OK) + { + return NGX_ERROR; + } +#endif + *rpeerp = &peer[n]; rpeerp = &peer[n].next; n++; @@ -211,6 +225,14 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, peer[n].down = server[i].down; peer[n].server = server[i].name; +#if (NGX_HTTP_UPSTREAM_SID) + if (ngx_http_upstream_create_sid(cf, &peer[n], &server[i].sid) + != NGX_OK) + { + return NGX_ERROR; + } +#endif + *peerp = &peer[n]; peerp = &peer[n].next; n++; @@ -316,6 +338,14 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, peer[n].down = server[i].down; peer[n].server = server[i].name; +#if (NGX_HTTP_UPSTREAM_SID) + if (ngx_http_upstream_create_sid(cf, &peer[n], &server[i].sid) + != NGX_OK) + { + return NGX_ERROR; + } +#endif + *rpeerp = &peer[n]; rpeerp = &peer[n].next; n++; @@ -337,6 +367,14 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, peer[n].down = server[i].down; peer[n].server = server[i].name; +#if (NGX_HTTP_UPSTREAM_SID) + if (ngx_http_upstream_create_sid(cf, &peer[n], &server[i].sid) + != NGX_OK) + { + return NGX_ERROR; + } +#endif + *peerp = &peer[n]; peerp = &peer[n].next; n++; @@ -416,6 +454,63 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, } +#if (NGX_HTTP_UPSTREAM_SID) + +static ngx_int_t +ngx_http_upstream_create_sid(ngx_conf_t *cf, ngx_http_upstream_rr_peer_t *peer, + ngx_str_t *route) +{ + if (route->len) { + peer->route = 1; + peer->sid = *route; + return NGX_OK; + } + + peer->sid.data = ngx_pnalloc(cf->pool, NGX_HTTP_UPSTREAM_SID_LEN); + if (peer->sid.data == NULL) { + return NGX_ERROR; + } + + ngx_http_upstream_init_round_robin_sid(peer, NULL); + + return NGX_OK; +} + + +void +ngx_http_upstream_init_round_robin_sid(ngx_http_upstream_rr_peer_t *peer, + ngx_str_t *route) +{ + u_char hash[16]; + ngx_md5_t md5; + + if (route && route->len) { + peer->route = 1; + peer->sid.len = route->len; + ngx_memcpy(peer->sid.data, route->data, route->len); + return; + } + + peer->route = 0; + + /* SID is the MD5 hash of a printable socket address */ + + if (peer->name.len == 0) { + peer->sid.len = 0; + return; + } + + ngx_md5_init(&md5); + ngx_md5_update(&md5, peer->name.data, peer->name.len); + ngx_md5_final(hash, &md5); + + ngx_hex_dump(peer->sid.data, hash, 16); + peer->sid.len = NGX_HTTP_UPSTREAM_SID_LEN; +} + +#endif + + ngx_int_t ngx_http_upstream_init_round_robin_peer(ngx_http_request_t *r, ngx_http_upstream_srv_conf_t *us) @@ -641,7 +736,7 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data) /* there are several peers */ - peer = ngx_http_upstream_get_peer(rrp); + peer = ngx_http_upstream_get_peer(rrp, pc); if (peer == NULL) { goto failed; @@ -656,6 +751,10 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data) pc->socklen = peer->socklen; pc->name = &peer->name; +#if (NGX_HTTP_UPSTREAM_SID) + pc->sid = &peer->sid; +#endif + peer->conns++; ngx_http_upstream_rr_peers_unlock(peers); @@ -701,7 +800,8 @@ busy: static ngx_http_upstream_rr_peer_t * -ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp) +ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp, + ngx_peer_connection_t *pc) { time_t now; uintptr_t m; @@ -709,6 +809,12 @@ ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp) ngx_uint_t i, n, p; ngx_http_upstream_rr_peer_t *peer, *best; +#if (NGX_HTTP_UPSTREAM_SID) + ngx_int_t low_limit; + ngx_uint_t st_p; + ngx_http_upstream_rr_peer_t *st_peer; +#endif + now = ngx_time(); best = NULL; @@ -718,6 +824,29 @@ ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp) p = 0; #endif +#if (NGX_HTTP_UPSTREAM_SID) + st_peer = ngx_http_upstream_get_rr_peer_by_sid(rrp, pc->hint, &p, 0); + + if (st_peer) { + + low_limit = -((ngx_int_t)(rrp->peers->total_weight - st_peer->weight)); + + /* + * note: current code accounts only one sticky request in a row, if it + * is required to account more, multiply low_limit by N below + */ + if (st_peer->current_weight <= low_limit) { + + /* do not update weights if the limit exceeded */ + best = st_peer; + goto best_chosen; + } + /* else: proceed to reweight with existing st_peer */ + } + + st_p = p; +#endif + for (peer = rrp->peers->peer, i = 0; peer; peer = peer->next, i++) @@ -757,10 +886,25 @@ ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp) } } +#if (NGX_HTTP_UPSTREAM_SID) + /* prefer peer chosen by sticky to best from RR */ + + if (st_peer) { + best = st_peer; + p = st_p; + } +#endif + if (best == NULL) { return NULL; } + best->current_weight -= total; + +#if (NGX_HTTP_UPSTREAM_SID) +best_chosen: +#endif + rrp->current = best; ngx_http_upstream_rr_peer_ref(rrp->peers, best); @@ -769,8 +913,6 @@ ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp) rrp->tried[n] |= m; - best->current_weight -= total; - if (now - best->checked > best->fail_timeout) { best->checked = now; } @@ -779,6 +921,77 @@ ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp) } +#if (NGX_HTTP_UPSTREAM_SID) + +ngx_http_upstream_rr_peer_t * +ngx_http_upstream_get_rr_peer_by_sid(ngx_http_upstream_rr_peer_data_t *rrp, + ngx_str_t *hint, ngx_uint_t *p, ngx_uint_t lock) +{ + uintptr_t m; + ngx_uint_t i, n; + ngx_http_upstream_rr_peer_t *peer; + + if (hint == NULL) { + return NULL; + } + + for (peer = rrp->peers->peer, i = 0; + peer; + peer = peer->next, i++) + { + + if (peer->sid.len == hint->len + && ngx_memcmp(peer->sid.data, hint->data, hint->len) == 0) + { + goto found; + } + } + + return NULL; + +found: + + n = i / (8 * sizeof(uintptr_t)); + m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); + + if (rrp->tried[n] & m) { + return NULL; + } + + if (lock) { + ngx_http_upstream_rr_peer_lock(rrp->peers, peer); + } + + if (peer->down) { + goto failed; + } + + if (peer->max_fails + && peer->fails >= peer->max_fails + && ngx_time() - peer->checked <= peer->fail_timeout) + { + goto failed; + } + + if (peer->max_conns && peer->conns >= peer->max_conns) { + goto failed; + } + + *p = i; + return peer; + +failed: + + if (lock) { + ngx_http_upstream_rr_peer_unlock(rrp->peers, peer); + } + + return NULL; +} + +#endif + + void ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data, ngx_uint_t state) diff --git a/src/http/ngx_http_upstream_round_robin.h b/src/http/ngx_http_upstream_round_robin.h index 2f0a51cdf..b5678ada1 100644 --- a/src/http/ngx_http_upstream_round_robin.h +++ b/src/http/ngx_http_upstream_round_robin.h @@ -14,6 +14,11 @@ #include <ngx_http.h> +#if (NGX_HTTP_UPSTREAM_SID) +#define NGX_HTTP_UPSTREAM_SID_LEN 32 /* md5 in hex */ +#endif + + typedef struct ngx_http_upstream_rr_peers_s ngx_http_upstream_rr_peers_t; typedef struct ngx_http_upstream_rr_peer_s ngx_http_upstream_rr_peer_t; @@ -62,6 +67,10 @@ struct ngx_http_upstream_rr_peer_s { int ssl_session_len; #endif +#if (NGX_HTTP_UPSTREAM_SID || NGX_COMPAT) + unsigned route:1; +#endif + #if (NGX_HTTP_UPSTREAM_ZONE) unsigned zombie:1; @@ -70,9 +79,13 @@ struct ngx_http_upstream_rr_peer_s { ngx_http_upstream_host_t *host; #endif +#if (NGX_HTTP_UPSTREAM_SID || NGX_COMPAT) + ngx_str_t sid; +#endif + ngx_http_upstream_rr_peer_t *next; - NGX_COMPAT_BEGIN(15) + NGX_COMPAT_BEGIN(13) NGX_COMPAT_END }; @@ -151,6 +164,9 @@ ngx_http_upstream_rr_peer_free_locked(ngx_http_upstream_rr_peers_t *peers, ngx_slab_free_locked(peers->shpool, peer->sockaddr); ngx_slab_free_locked(peers->shpool, peer->name.data); +#if (NGX_HTTP_UPSTREAM_SID) + ngx_slab_free_locked(peers->shpool, peer->sid.data); +#endif if (peer->server.data) { ngx_slab_free_locked(peers->shpool, peer->server.data); @@ -235,5 +251,19 @@ void ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc, void *data); #endif +#if (NGX_HTTP_UPSTREAM_SID) + +#define ngx_http_upstream_copy_round_robin_sid(dst, src) \ + ngx_http_upstream_init_round_robin_sid(dst, \ + (src)->route ? &(src)->sid : NULL) + +void ngx_http_upstream_init_round_robin_sid(ngx_http_upstream_rr_peer_t *peer, + ngx_str_t *route); +ngx_http_upstream_rr_peer_t *ngx_http_upstream_get_rr_peer_by_sid( + ngx_http_upstream_rr_peer_data_t *rrp, ngx_str_t *hint, ngx_uint_t *p, + ngx_uint_t lock); + +#endif + #endif /* _NGX_HTTP_UPSTREAM_ROUND_ROBIN_H_INCLUDED_ */ |