Klaus Demo nginx / eaf3544
Merge of r4655, r4656, r4657, r4695, r4696: upstream changes. *) Upstream: least_conn balancer module. *) Upstream: weights and IPv6 support in ip_hash balancer. *) Upstream keepalive: "single" parameter deprecated. Maxim Dounin 9 years ago
8 changed file(s) with 480 addition(s) and 49 deletion(s). Raw diff Collapse all Expand all
+5
-0
auto/modules less more
344344 HTTP_SRCS="$HTTP_SRCS $HTTP_UPSTREAM_IP_HASH_SRCS"
345345 fi
346346
347 if [ $HTTP_UPSTREAM_LEAST_CONN = YES ]; then
348 HTTP_MODULES="$HTTP_MODULES $HTTP_UPSTREAM_LEAST_CONN_MODULE"
349 HTTP_SRCS="$HTTP_SRCS $HTTP_UPSTREAM_LEAST_CONN_SRCS"
350 fi
351
347352 if [ $HTTP_UPSTREAM_KEEPALIVE = YES ]; then
348353 HTTP_MODULES="$HTTP_MODULES $HTTP_UPSTREAM_KEEPALIVE_MODULE"
349354 HTTP_SRCS="$HTTP_SRCS $HTTP_UPSTREAM_KEEPALIVE_SRCS"
+3
-0
auto/options less more
9595 HTTP_MP4=NO
9696 HTTP_GZIP_STATIC=NO
9797 HTTP_UPSTREAM_IP_HASH=YES
98 HTTP_UPSTREAM_LEAST_CONN=YES
9899 HTTP_UPSTREAM_KEEPALIVE=YES
99100
100101 # STUB
242243 --without-http_empty_gif_module) HTTP_EMPTY_GIF=NO ;;
243244 --without-http_browser_module) HTTP_BROWSER=NO ;;
244245 --without-http_upstream_ip_hash_module) HTTP_UPSTREAM_IP_HASH=NO ;;
246 --without-http_upstream_least_conn_module)
247 HTTP_UPSTREAM_LEAST_CONN=NO ;;
245248 --without-http_upstream_keepalive_module) HTTP_UPSTREAM_KEEPALIVE=NO ;;
246249
247250 --with-http_perl_module) HTTP_PERL=YES ;;
+5
-0
auto/sources less more
478478 HTTP_UPSTREAM_IP_HASH_SRCS=src/http/modules/ngx_http_upstream_ip_hash_module.c
479479
480480
481 HTTP_UPSTREAM_LEAST_CONN_MODULE=ngx_http_upstream_least_conn_module
482 HTTP_UPSTREAM_LEAST_CONN_SRCS=" \
483 src/http/modules/ngx_http_upstream_least_conn_module.c"
484
485
481486 HTTP_UPSTREAM_KEEPALIVE_MODULE=ngx_http_upstream_keepalive_module
482487 HTTP_UPSTREAM_KEEPALIVE_SRCS=" \
483488 src/http/modules/ngx_http_upstream_keepalive_module.c"
+45
-18
src/http/modules/ngx_http_upstream_ip_hash_module.c less more
1515
1616 ngx_uint_t hash;
1717
18 u_char addr[3];
18 u_char addrlen;
19 u_char *addr;
1920
2021 u_char tries;
2122
7576 };
7677
7778
78 ngx_int_t
79 static u_char ngx_http_upstream_ip_hash_pseudo_addr[3];
80
81
82 static ngx_int_t
7983 ngx_http_upstream_init_ip_hash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us)
8084 {
8185 if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) {
9296 ngx_http_upstream_init_ip_hash_peer(ngx_http_request_t *r,
9397 ngx_http_upstream_srv_conf_t *us)
9498 {
95 u_char *p;
9699 struct sockaddr_in *sin;
100 #if (NGX_HAVE_INET6)
101 struct sockaddr_in6 *sin6;
102 #endif
97103 ngx_http_upstream_ip_hash_peer_data_t *iphp;
98104
99105 iphp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_ip_hash_peer_data_t));
109115
110116 r->upstream->peer.get = ngx_http_upstream_get_ip_hash_peer;
111117
112 /* AF_INET only */
113
114 if (r->connection->sockaddr->sa_family == AF_INET) {
115
118 switch (r->connection->sockaddr->sa_family) {
119
120 case AF_INET:
116121 sin = (struct sockaddr_in *) r->connection->sockaddr;
117 p = (u_char *) &sin->sin_addr.s_addr;
118 iphp->addr[0] = p[0];
119 iphp->addr[1] = p[1];
120 iphp->addr[2] = p[2];
121
122 } else {
123 iphp->addr[0] = 0;
124 iphp->addr[1] = 0;
125 iphp->addr[2] = 0;
122 iphp->addr = (u_char *) &sin->sin_addr.s_addr;
123 iphp->addrlen = 3;
124 break;
125
126 #if (NGX_HAVE_INET6)
127 case AF_INET6:
128 sin6 = (struct sockaddr_in6 *) r->connection->sockaddr;
129 iphp->addr = (u_char *) &sin6->sin6_addr.s6_addr;
130 iphp->addrlen = 16;
131 break;
132 #endif
133
134 default:
135 iphp->addr = ngx_http_upstream_ip_hash_pseudo_addr;
136 iphp->addrlen = 3;
126137 }
127138
128139 iphp->hash = 89;
139150 ngx_http_upstream_ip_hash_peer_data_t *iphp = data;
140151
141152 time_t now;
153 ngx_int_t w;
142154 uintptr_t m;
143155 ngx_uint_t i, n, p, hash;
144156 ngx_http_upstream_rr_peer_t *peer;
161173
162174 for ( ;; ) {
163175
164 for (i = 0; i < 3; i++) {
176 for (i = 0; i < iphp->addrlen; i++) {
165177 hash = (hash * 113 + iphp->addr[i]) % 6271;
166178 }
167179
168 p = hash % iphp->rrp.peers->number;
180 if (!iphp->rrp.peers->weighted) {
181 p = hash % iphp->rrp.peers->number;
182
183 } else {
184 w = hash % iphp->rrp.peers->total_weight;
185
186 for (i = 0; i < iphp->rrp.peers->number; i++) {
187 w -= iphp->rrp.peers->peer[i].weight;
188 if (w < 0) {
189 break;
190 }
191 }
192
193 p = i;
194 }
169195
170196 n = p / (8 * sizeof(uintptr_t));
171197 m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
228254 uscf->peer.init_upstream = ngx_http_upstream_init_ip_hash;
229255
230256 uscf->flags = NGX_HTTP_UPSTREAM_CREATE
257 |NGX_HTTP_UPSTREAM_WEIGHT
231258 |NGX_HTTP_UPSTREAM_MAX_FAILS
232259 |NGX_HTTP_UPSTREAM_FAIL_TIMEOUT
233260 |NGX_HTTP_UPSTREAM_DOWN;
+4
-29
src/http/modules/ngx_http_upstream_keepalive_module.c less more
1111
1212 typedef struct {
1313 ngx_uint_t max_cached;
14 ngx_uint_t single; /* unsigned:1 */
1514
1615 ngx_queue_t cache;
1716 ngx_queue_t free;
222221
223222 kp->failed = 0;
224223
225 /* single pool of cached connections */
226
227 if (kp->conf->single && !ngx_queue_empty(&kp->conf->cache)) {
228
229 q = ngx_queue_head(&kp->conf->cache);
230
231 item = ngx_queue_data(q, ngx_http_upstream_keepalive_cache_t, queue);
232 c = item->connection;
233
234 ngx_queue_remove(q);
235 ngx_queue_insert_head(&kp->conf->free, q);
236
237 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
238 "get keepalive peer: using connection %p", c);
239
240 c->idle = 0;
241 c->log = pc->log;
242 c->read->log = pc->log;
243 c->write->log = pc->log;
244 c->pool->log = pc->log;
245
246 pc->connection = c;
247 pc->cached = 1;
248
249 return NGX_DONE;
250 }
224 /* ask balancer */
251225
252226 rc = kp->original_get_peer(pc, kp->data);
253227
254 if (kp->conf->single || rc != NGX_OK) {
228 if (rc != NGX_OK) {
255229 return rc;
256230 }
257231
551525 for (i = 2; i < cf->args->nelts; i++) {
552526
553527 if (ngx_strcmp(value[i].data, "single") == 0) {
554 kcf->single = 1;
528 ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
529 "the \"single\" parameter is deprecated");
555530 continue;
556531 }
557532
+402
-0
src/http/modules/ngx_http_upstream_least_conn_module.c less more
0
1 /*
2 * Copyright (C) Maxim Dounin
3 * Copyright (C) Nginx, Inc.
4 */
5
6
7 #include <ngx_config.h>
8 #include <ngx_core.h>
9 #include <ngx_http.h>
10
11
12 typedef struct {
13 ngx_uint_t *conns;
14 } ngx_http_upstream_least_conn_conf_t;
15
16
17 typedef struct {
18 /* the round robin data must be first */
19 ngx_http_upstream_rr_peer_data_t rrp;
20
21 ngx_uint_t *conns;
22
23 ngx_event_get_peer_pt get_rr_peer;
24 ngx_event_free_peer_pt free_rr_peer;
25 } ngx_http_upstream_lc_peer_data_t;
26
27
28 static ngx_int_t ngx_http_upstream_init_least_conn_peer(ngx_http_request_t *r,
29 ngx_http_upstream_srv_conf_t *us);
30 static ngx_int_t ngx_http_upstream_get_least_conn_peer(
31 ngx_peer_connection_t *pc, void *data);
32 static void ngx_http_upstream_free_least_conn_peer(ngx_peer_connection_t *pc,
33 void *data, ngx_uint_t state);
34 static void *ngx_http_upstream_least_conn_create_conf(ngx_conf_t *cf);
35 static char *ngx_http_upstream_least_conn(ngx_conf_t *cf, ngx_command_t *cmd,
36 void *conf);
37
38
39 static ngx_command_t ngx_http_upstream_least_conn_commands[] = {
40
41 { ngx_string("least_conn"),
42 NGX_HTTP_UPS_CONF|NGX_CONF_NOARGS,
43 ngx_http_upstream_least_conn,
44 0,
45 0,
46 NULL },
47
48 ngx_null_command
49 };
50
51
52 static ngx_http_module_t ngx_http_upstream_least_conn_module_ctx = {
53 NULL, /* preconfiguration */
54 NULL, /* postconfiguration */
55
56 NULL, /* create main configuration */
57 NULL, /* init main configuration */
58
59 ngx_http_upstream_least_conn_create_conf, /* create server configuration */
60 NULL, /* merge server configuration */
61
62 NULL, /* create location configuration */
63 NULL /* merge location configuration */
64 };
65
66
67 ngx_module_t ngx_http_upstream_least_conn_module = {
68 NGX_MODULE_V1,
69 &ngx_http_upstream_least_conn_module_ctx, /* module context */
70 ngx_http_upstream_least_conn_commands, /* module directives */
71 NGX_HTTP_MODULE, /* module type */
72 NULL, /* init master */
73 NULL, /* init module */
74 NULL, /* init process */
75 NULL, /* init thread */
76 NULL, /* exit thread */
77 NULL, /* exit process */
78 NULL, /* exit master */
79 NGX_MODULE_V1_PADDING
80 };
81
82
83 ngx_int_t
84 ngx_http_upstream_init_least_conn(ngx_conf_t *cf,
85 ngx_http_upstream_srv_conf_t *us)
86 {
87 ngx_uint_t n;
88 ngx_http_upstream_rr_peers_t *peers;
89 ngx_http_upstream_least_conn_conf_t *lcf;
90
91 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cf->log, 0,
92 "init least conn");
93
94 if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) {
95 return NGX_ERROR;
96 }
97
98 peers = us->peer.data;
99
100 n = peers->number;
101
102 if (peers->next) {
103 n += peers->next->number;
104 }
105
106 lcf = ngx_http_conf_upstream_srv_conf(us,
107 ngx_http_upstream_least_conn_module);
108
109 lcf->conns = ngx_pcalloc(cf->pool, sizeof(ngx_uint_t) * n);
110 if (lcf->conns == NULL) {
111 return NGX_ERROR;
112 }
113
114 us->peer.init = ngx_http_upstream_init_least_conn_peer;
115
116 return NGX_OK;
117 }
118
119
120 static ngx_int_t
121 ngx_http_upstream_init_least_conn_peer(ngx_http_request_t *r,
122 ngx_http_upstream_srv_conf_t *us)
123 {
124 ngx_http_upstream_lc_peer_data_t *lcp;
125 ngx_http_upstream_least_conn_conf_t *lcf;
126
127 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
128 "init least conn peer");
129
130 lcf = ngx_http_conf_upstream_srv_conf(us,
131 ngx_http_upstream_least_conn_module);
132
133 lcp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_lc_peer_data_t));
134 if (lcp == NULL) {
135 return NGX_ERROR;
136 }
137
138 lcp->conns = lcf->conns;
139
140 r->upstream->peer.data = &lcp->rrp;
141
142 if (ngx_http_upstream_init_round_robin_peer(r, us) != NGX_OK) {
143 return NGX_ERROR;
144 }
145
146 r->upstream->peer.get = ngx_http_upstream_get_least_conn_peer;
147 r->upstream->peer.free = ngx_http_upstream_free_least_conn_peer;
148
149 lcp->get_rr_peer = ngx_http_upstream_get_round_robin_peer;
150 lcp->free_rr_peer = ngx_http_upstream_free_round_robin_peer;
151
152 return NGX_OK;
153 }
154
155
156 static ngx_int_t
157 ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data)
158 {
159 ngx_http_upstream_lc_peer_data_t *lcp = data;
160
161 time_t now;
162 uintptr_t m;
163 ngx_int_t rc, total;
164 ngx_uint_t i, n, p, many;
165 ngx_http_upstream_rr_peer_t *peer, *best;
166 ngx_http_upstream_rr_peers_t *peers;
167
168 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
169 "get least conn peer, try: %ui", pc->tries);
170
171 if (lcp->rrp.peers->single) {
172 return lcp->get_rr_peer(pc, &lcp->rrp);
173 }
174
175 pc->cached = 0;
176 pc->connection = NULL;
177
178 now = ngx_time();
179
180 peers = lcp->rrp.peers;
181
182 best = NULL;
183 total = 0;
184
185 #if (NGX_SUPPRESS_WARN)
186 many = 0;
187 p = 0;
188 #endif
189
190 for (i = 0; i < peers->number; i++) {
191
192 n = i / (8 * sizeof(uintptr_t));
193 m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
194
195 if (lcp->rrp.tried[n] & m) {
196 continue;
197 }
198
199 peer = &peers->peer[i];
200
201 if (peer->down) {
202 continue;
203 }
204
205 if (peer->max_fails
206 && peer->fails >= peer->max_fails
207 && now - peer->checked <= peer->fail_timeout)
208 {
209 continue;
210 }
211
212 /*
213 * select peer with least number of connections; if there are
214 * multiple peers with the same number of connections, select
215 * based on round-robin
216 */
217
218 if (best == NULL
219 || lcp->conns[i] * best->weight < lcp->conns[p] * peer->weight)
220 {
221 best = peer;
222 many = 0;
223 p = i;
224
225 } else if (lcp->conns[i] * best->weight
226 == lcp->conns[p] * peer->weight)
227 {
228 many = 1;
229 }
230 }
231
232 if (best == NULL) {
233 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0,
234 "get least conn peer, no peer found");
235
236 goto failed;
237 }
238
239 if (many) {
240 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0,
241 "get least conn peer, many");
242
243 for (i = p; i < peers->number; i++) {
244
245 n = i / (8 * sizeof(uintptr_t));
246 m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
247
248 if (lcp->rrp.tried[n] & m) {
249 continue;
250 }
251
252 peer = &peers->peer[i];
253
254 if (peer->down) {
255 continue;
256 }
257
258 if (lcp->conns[i] * best->weight != lcp->conns[p] * peer->weight) {
259 continue;
260 }
261
262 if (peer->max_fails
263 && peer->fails >= peer->max_fails
264 && now - peer->checked <= peer->fail_timeout)
265 {
266 continue;
267 }
268
269 peer->current_weight += peer->effective_weight;
270 total += peer->effective_weight;
271
272 if (peer->effective_weight < peer->weight) {
273 peer->effective_weight++;
274 }
275
276 if (peer->current_weight > best->current_weight) {
277 best = peer;
278 p = i;
279 }
280 }
281 }
282
283 best->current_weight -= total;
284 best->checked = now;
285
286 pc->sockaddr = best->sockaddr;
287 pc->socklen = best->socklen;
288 pc->name = &best->name;
289
290 lcp->rrp.current = p;
291
292 n = p / (8 * sizeof(uintptr_t));
293 m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
294
295 lcp->rrp.tried[n] |= m;
296 lcp->conns[p]++;
297
298 if (pc->tries == 1 && peers->next) {
299 pc->tries += peers->next->number;
300 }
301
302 return NGX_OK;
303
304 failed:
305
306 if (peers->next) {
307 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0,
308 "get least conn peer, backup servers");
309
310 lcp->conns += peers->number;
311
312 lcp->rrp.peers = peers->next;
313 pc->tries = lcp->rrp.peers->number;
314
315 n = lcp->rrp.peers->number / (8 * sizeof(uintptr_t)) + 1;
316 for (i = 0; i < n; i++) {
317 lcp->rrp.tried[i] = 0;
318 }
319
320 rc = ngx_http_upstream_get_least_conn_peer(pc, lcp);
321
322 if (rc != NGX_BUSY) {
323 return rc;
324 }
325 }
326
327 /* all peers failed, mark them as live for quick recovery */
328
329 for (i = 0; i < peers->number; i++) {
330 peers->peer[i].fails = 0;
331 }
332
333 pc->name = peers->name;
334
335 return NGX_BUSY;
336 }
337
338
339 static void
340 ngx_http_upstream_free_least_conn_peer(ngx_peer_connection_t *pc,
341 void *data, ngx_uint_t state)
342 {
343 ngx_http_upstream_lc_peer_data_t *lcp = data;
344
345 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
346 "free least conn peer %ui %ui", pc->tries, state);
347
348 if (lcp->rrp.peers->single) {
349 lcp->free_rr_peer(pc, &lcp->rrp, state);
350 return;
351 }
352
353 if (state == 0 && pc->tries == 0) {
354 return;
355 }
356
357 lcp->conns[lcp->rrp.current]--;
358
359 lcp->free_rr_peer(pc, &lcp->rrp, state);
360 }
361
362
363 static void *
364 ngx_http_upstream_least_conn_create_conf(ngx_conf_t *cf)
365 {
366 ngx_http_upstream_least_conn_conf_t *conf;
367
368 conf = ngx_pcalloc(cf->pool,
369 sizeof(ngx_http_upstream_least_conn_conf_t));
370 if (conf == NULL) {
371 return NULL;
372 }
373
374 /*
375 * set by ngx_pcalloc():
376 *
377 * conf->conns = NULL;
378 */
379
380 return conf;
381 }
382
383
384 static char *
385 ngx_http_upstream_least_conn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
386 {
387 ngx_http_upstream_srv_conf_t *uscf;
388
389 uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module);
390
391 uscf->peer.init_upstream = ngx_http_upstream_init_least_conn;
392
393 uscf->flags = NGX_HTTP_UPSTREAM_CREATE
394 |NGX_HTTP_UPSTREAM_WEIGHT
395 |NGX_HTTP_UPSTREAM_MAX_FAILS
396 |NGX_HTTP_UPSTREAM_FAIL_TIMEOUT
397 |NGX_HTTP_UPSTREAM_DOWN
398 |NGX_HTTP_UPSTREAM_BACKUP;
399
400 return NGX_CONF_OK;
401 }
+11
-1
src/http/ngx_http_upstream_round_robin.c less more
2929 ngx_http_upstream_srv_conf_t *us)
3030 {
3131 ngx_url_t u;
32 ngx_uint_t i, j, n;
32 ngx_uint_t i, j, n, w;
3333 ngx_http_upstream_server_t *server;
3434 ngx_http_upstream_rr_peers_t *peers, *backup;
3535
3939 server = us->servers->elts;
4040
4141 n = 0;
42 w = 0;
4243
4344 for (i = 0; i < us->servers->nelts; i++) {
4445 if (server[i].backup) {
4647 }
4748
4849 n += server[i].naddrs;
50 w += server[i].naddrs * server[i].weight;
4951 }
5052
5153 if (n == 0) {
6365
6466 peers->single = (n == 1);
6567 peers->number = n;
68 peers->weighted = (w != n);
69 peers->total_weight = w;
6670 peers->name = &us->host;
6771
6872 n = 0;
9599 /* backup servers */
96100
97101 n = 0;
102 w = 0;
98103
99104 for (i = 0; i < us->servers->nelts; i++) {
100105 if (!server[i].backup) {
102107 }
103108
104109 n += server[i].naddrs;
110 w += server[i].naddrs * server[i].weight;
105111 }
106112
107113 if (n == 0) {
117123 peers->single = 0;
118124 backup->single = 0;
119125 backup->number = n;
126 backup->weighted = (w != n);
127 backup->total_weight = w;
120128 backup->name = &us->host;
121129
122130 n = 0;
184192
185193 peers->single = (n == 1);
186194 peers->number = n;
195 peers->weighted = 0;
196 peers->total_weight = n;
187197 peers->name = &us->host;
188198
189199 for (i = 0; i < u.naddrs; i++) {
+5
-1
src/http/ngx_http_upstream_round_robin.h less more
4040 typedef struct ngx_http_upstream_rr_peers_s ngx_http_upstream_rr_peers_t;
4141
4242 struct ngx_http_upstream_rr_peers_s {
43 ngx_uint_t single; /* unsigned single:1; */
4443 ngx_uint_t number;
4544 ngx_uint_t last_cached;
4645
4746 /* ngx_mutex_t *mutex; */
4847 ngx_connection_t **cached;
48
49 ngx_uint_t total_weight;
50
51 unsigned single:1;
52 unsigned weighted:1;
4953
5054 ngx_str_t *name;
5155