Klaus Demo nginx / 5429140
HTTP/2: implemented preread buffer for request body (closes #959). Previously, the stream's window was kept zero in order to prevent a client from sending the request body before it was requested (see 887cca40ba6a for details). Until such initial window was acknowledged all requests with data were rejected (see 0aa07850922f for details). That approach revealed a number of problems: 1. Some clients (notably MS IE/Edge, Safari, iOS applications) show an error or even crash if a stream is rejected; 2. This requires at least one RTT for every request with body before the client receives window update and able to send data. To overcome these problems the new directive "http2_body_preread_size" is introduced. It sets the initial window and configures a special per stream preread buffer that is used to save all incoming data before the body is requested and processed. If the directive's value is lower than the default initial window (65535), as previously, all streams with data will be rejected until the new window is acknowledged. Otherwise, no special processing is used and all requests with data are welcome right from the connection start. The default value is chosen to be 64k, which is bigger than the default initial window. Setting it to zero is fully complaint to the previous behavior. Valentin Bartenev 6 years ago
4 changed file(s) with 155 addition(s) and 52 deletion(s). Raw diff Collapse all Expand all
+118
-52
src/http/v2/ngx_http_v2.c less more
4646 #define NGX_HTTP_V2_FRAME_BUFFER_SIZE 24
4747
4848 #define NGX_HTTP_V2_DEFAULT_FRAME_SIZE (1 << 14)
49
50 #define NGX_HTTP_V2_MAX_WINDOW ((1U << 31) - 1)
51 #define NGX_HTTP_V2_DEFAULT_WINDOW 65535
52
53 #define NGX_HTTP_V2_INITIAL_WINDOW 0
5449
5550 #define NGX_HTTP_V2_ROOT (void *) -1
5651
878873 return ngx_http_v2_state_skip_padded(h2c, pos, end);
879874 }
880875
881 stream->in_closed = h2c->state.flags & NGX_HTTP_V2_END_STREAM_FLAG;
882
883876 h2c->state.stream = stream;
884877
885878 return ngx_http_v2_state_read_data(h2c, pos, end);
890883 ngx_http_v2_state_read_data(ngx_http_v2_connection_t *h2c, u_char *pos,
891884 u_char *end)
892885 {
893 size_t size;
894 ngx_int_t rc;
895 ngx_uint_t last;
896 ngx_http_v2_stream_t *stream;
886 size_t size;
887 ngx_buf_t *buf;
888 ngx_int_t rc;
889 ngx_http_request_t *r;
890 ngx_http_v2_stream_t *stream;
891 ngx_http_v2_srv_conf_t *h2scf;
897892
898893 stream = h2c->state.stream;
899894
912907
913908 if (size >= h2c->state.length) {
914909 size = h2c->state.length;
915 last = stream->in_closed;
916
917 } else {
918 last = 0;
919 }
920
921 rc = ngx_http_v2_process_request_body(stream->request, pos, size, last);
922
923 if (rc != NGX_OK) {
924 stream->skip_data = 1;
925 ngx_http_finalize_request(stream->request, rc);
910 stream->in_closed = h2c->state.flags & NGX_HTTP_V2_END_STREAM_FLAG;
911 }
912
913 r = stream->request;
914
915 if (r->request_body) {
916 rc = ngx_http_v2_process_request_body(r, pos, size, stream->in_closed);
917
918 if (rc != NGX_OK) {
919 stream->skip_data = 1;
920 ngx_http_finalize_request(r, rc);
921 }
922
923 } else if (size) {
924 buf = stream->preread;
925
926 if (buf == NULL) {
927 h2scf = ngx_http_get_module_srv_conf(r, ngx_http_v2_module);
928
929 buf = ngx_create_temp_buf(r->pool, h2scf->preread_size);
930 if (buf == NULL) {
931 return ngx_http_v2_connection_error(h2c,
932 NGX_HTTP_V2_INTERNAL_ERROR);
933 }
934
935 stream->preread = buf;
936 }
937
938 if (size > (size_t) (buf->end - buf->last)) {
939 ngx_log_error(NGX_LOG_ALERT, h2c->connection->log, 0,
940 "http2 preread buffer overflow");
941 return ngx_http_v2_connection_error(h2c,
942 NGX_HTTP_V2_INTERNAL_ERROR);
943 }
944
945 buf->last = ngx_cpymem(buf->last, pos, size);
926946 }
927947
928948 pos += size;
10571077 goto rst_stream;
10581078 }
10591079
1060 if (!h2c->settings_ack && !(h2c->state.flags & NGX_HTTP_V2_END_STREAM_FLAG))
1080 if (!h2c->settings_ack
1081 && !(h2c->state.flags & NGX_HTTP_V2_END_STREAM_FLAG)
1082 && h2scf->preread_size < NGX_HTTP_V2_DEFAULT_WINDOW)
10611083 {
10621084 ngx_log_error(NGX_LOG_INFO, h2c->connection->log, 0,
10631085 "client sent stream with data "
24332455
24342456 buf->last = ngx_http_v2_write_uint16(buf->last,
24352457 NGX_HTTP_V2_INIT_WINDOW_SIZE_SETTING);
2436 buf->last = ngx_http_v2_write_uint32(buf->last,
2437 NGX_HTTP_V2_INITIAL_WINDOW);
2458 buf->last = ngx_http_v2_write_uint32(buf->last, h2scf->preread_size);
24382459
24392460 buf->last = ngx_http_v2_write_uint16(buf->last,
24402461 NGX_HTTP_V2_MAX_FRAME_SIZE_SETTING);
26422663 ngx_http_log_ctx_t *ctx;
26432664 ngx_http_request_t *r;
26442665 ngx_http_v2_stream_t *stream;
2666 ngx_http_v2_srv_conf_t *h2scf;
26452667 ngx_http_core_srv_conf_t *cscf;
26462668
26472669 fc = h2c->free_fake_connections;
27552777 stream->request = r;
27562778 stream->connection = h2c;
27572779
2780 h2scf = ngx_http_get_module_srv_conf(r, ngx_http_v2_module);
2781
27582782 stream->send_window = h2c->init_window;
2759 stream->recv_window = NGX_HTTP_V2_INITIAL_WINDOW;
2783 stream->recv_window = h2scf->preread_size;
27602784
27612785 h2c->processing++;
27622786
34103434 ngx_http_client_body_handler_pt post_handler)
34113435 {
34123436 off_t len;
3437 size_t size;
3438 ngx_buf_t *buf;
3439 ngx_int_t rc;
34133440 ngx_http_v2_stream_t *stream;
3441 ngx_http_v2_srv_conf_t *h2scf;
34143442 ngx_http_request_body_t *rb;
34153443 ngx_http_core_loc_conf_t *clcf;
34163444 ngx_http_v2_connection_t *h2c;
34433471
34443472 r->request_body = rb;
34453473
3474 h2scf = ngx_http_get_module_srv_conf(r, ngx_http_v2_module);
34463475 clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
34473476
34483477 len = r->headers_in.content_length_n;
34493478
34503479 if (r->request_body_no_buffering && !stream->in_closed) {
3451 r->request_body_in_file_only = 0;
34523480
34533481 if (len < 0 || len > (off_t) clcf->client_body_buffer_size) {
34543482 len = clcf->client_body_buffer_size;
34553483 }
34563484
3485 /*
3486 * We need a room to store data up to the stream's initial window size,
3487 * at least until this window will be exhausted.
3488 */
3489
3490 if (len < (off_t) h2scf->preread_size) {
3491 len = h2scf->preread_size;
3492 }
3493
34573494 if (len > NGX_HTTP_V2_MAX_WINDOW) {
34583495 len = NGX_HTTP_V2_MAX_WINDOW;
34593496 }
3460 }
3461
3462 if (len >= 0 && len <= (off_t) clcf->client_body_buffer_size
3463 && !r->request_body_in_file_only)
3497
3498 rb->buf = ngx_create_temp_buf(r->pool, (size_t) len);
3499
3500 } else if (len >= 0 && len <= (off_t) clcf->client_body_buffer_size
3501 && !r->request_body_in_file_only)
34643502 {
34653503 rb->buf = ngx_create_temp_buf(r->pool, (size_t) len);
34663504
34773515 return NGX_HTTP_INTERNAL_SERVER_ERROR;
34783516 }
34793517
3518 buf = stream->preread;
3519
34803520 if (stream->in_closed) {
34813521 r->request_body_no_buffering = 0;
3522
3523 if (buf) {
3524 rc = ngx_http_v2_process_request_body(r, buf->pos,
3525 buf->last - buf->pos, 1);
3526 ngx_pfree(r->pool, buf->start);
3527 return rc;
3528 }
3529
34823530 return ngx_http_v2_process_request_body(r, NULL, 0, 1);
34833531 }
34843532
3485 if (len) {
3486 if (r->request_body_no_buffering) {
3487 stream->recv_window = (size_t) len;
3488
3489 } else {
3490 stream->no_flow_control = 1;
3491 stream->recv_window = NGX_HTTP_V2_MAX_WINDOW;
3492 }
3493
3494 if (ngx_http_v2_send_window_update(stream->connection, stream->node->id,
3495 stream->recv_window)
3533 if (buf) {
3534 rc = ngx_http_v2_process_request_body(r, buf->pos,
3535 buf->last - buf->pos, 0);
3536
3537 ngx_pfree(r->pool, buf->start);
3538
3539 if (rc != NGX_OK) {
3540 stream->skip_data = 1;
3541 return rc;
3542 }
3543 }
3544
3545 if (r->request_body_no_buffering) {
3546 size = len - h2scf->preread_size;
3547
3548 } else {
3549 stream->no_flow_control = 1;
3550 size = NGX_HTTP_V2_MAX_WINDOW - stream->recv_window;
3551 }
3552
3553 if (size) {
3554 if (ngx_http_v2_send_window_update(stream->connection,
3555 stream->node->id, size)
34963556 == NGX_ERROR)
34973557 {
34983558 stream->skip_data = 1;
35073567 return NGX_HTTP_INTERNAL_SERVER_ERROR;
35083568 }
35093569 }
3510 }
3511
3512 ngx_add_timer(r->connection->read, clcf->client_body_timeout);
3570
3571 stream->recv_window += size;
3572 }
3573
3574 if (!buf) {
3575 ngx_add_timer(r->connection->read, clcf->client_body_timeout);
3576 }
35133577
35143578 r->read_event_handler = ngx_http_v2_read_client_request_body_handler;
35153579 r->write_event_handler = ngx_http_request_empty_handler;
35283592 ngx_http_request_body_t *rb;
35293593 ngx_http_core_loc_conf_t *clcf;
35303594
3595 fc = r->connection;
35313596 rb = r->request_body;
3532
3533 if (rb == NULL) {
3534 return NGX_OK;
3535 }
3536
3537 fc = r->connection;
35383597 buf = rb->buf;
35393598
35403599 if (size) {
37883847 window -= h2c->state.length;
37893848 }
37903849
3791 if (window == stream->recv_window) {
3850 if (window <= stream->recv_window) {
3851 if (window < stream->recv_window) {
3852 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
3853 "http2 negative window update");
3854 stream->skip_data = 1;
3855 return NGX_HTTP_INTERNAL_SERVER_ERROR;
3856 }
3857
37923858 return NGX_AGAIN;
37933859 }
37943860
+5
-0
src/http/v2/ngx_http_v2.h less more
4444 #define NGX_HTTP_V2_END_HEADERS_FLAG 0x04
4545 #define NGX_HTTP_V2_PADDED_FLAG 0x08
4646 #define NGX_HTTP_V2_PRIORITY_FLAG 0x20
47
48 #define NGX_HTTP_V2_MAX_WINDOW ((1U << 31) - 1)
49 #define NGX_HTTP_V2_DEFAULT_WINDOW 65535
4750
4851
4952 typedef struct ngx_http_v2_connection_s ngx_http_v2_connection_t;
173176 ssize_t send_window;
174177 size_t recv_window;
175178
179 ngx_buf_t *preread;
180
176181 ngx_http_v2_out_frame_t *free_frames;
177182 ngx_chain_t *free_frame_headers;
178183 ngx_chain_t *free_bufs;
+31
-0
src/http/v2/ngx_http_v2_module.c less more
2929 static char *ngx_http_v2_recv_buffer_size(ngx_conf_t *cf, void *post,
3030 void *data);
3131 static char *ngx_http_v2_pool_size(ngx_conf_t *cf, void *post, void *data);
32 static char *ngx_http_v2_preread_size(ngx_conf_t *cf, void *post, void *data);
3233 static char *ngx_http_v2_streams_index_mask(ngx_conf_t *cf, void *post,
3334 void *data);
3435 static char *ngx_http_v2_chunk_size(ngx_conf_t *cf, void *post, void *data);
4041 { ngx_http_v2_recv_buffer_size };
4142 static ngx_conf_post_t ngx_http_v2_pool_size_post =
4243 { ngx_http_v2_pool_size };
44 static ngx_conf_post_t ngx_http_v2_preread_size_post =
45 { ngx_http_v2_preread_size };
4346 static ngx_conf_post_t ngx_http_v2_streams_index_mask_post =
4447 { ngx_http_v2_streams_index_mask };
4548 static ngx_conf_post_t ngx_http_v2_chunk_size_post =
8285 NGX_HTTP_SRV_CONF_OFFSET,
8386 offsetof(ngx_http_v2_srv_conf_t, max_header_size),
8487 NULL },
88
89 { ngx_string("http2_body_preread_size"),
90 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1,
91 ngx_conf_set_size_slot,
92 NGX_HTTP_SRV_CONF_OFFSET,
93 offsetof(ngx_http_v2_srv_conf_t, preread_size),
94 &ngx_http_v2_preread_size_post },
8595
8696 { ngx_string("http2_streams_index_size"),
8797 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1,
315325 h2scf->max_field_size = NGX_CONF_UNSET_SIZE;
316326 h2scf->max_header_size = NGX_CONF_UNSET_SIZE;
317327
328 h2scf->preread_size = NGX_CONF_UNSET_SIZE;
329
318330 h2scf->streams_index_mask = NGX_CONF_UNSET_UINT;
319331
320332 h2scf->recv_timeout = NGX_CONF_UNSET_MSEC;
340352 ngx_conf_merge_size_value(conf->max_header_size, prev->max_header_size,
341353 16384);
342354
355 ngx_conf_merge_size_value(conf->preread_size, prev->preread_size, 65536);
356
343357 ngx_conf_merge_uint_value(conf->streams_index_mask,
344358 prev->streams_index_mask, 32 - 1);
345359
419433
420434
421435 static char *
436 ngx_http_v2_preread_size(ngx_conf_t *cf, void *post, void *data)
437 {
438 size_t *sp = data;
439
440 if (*sp > NGX_HTTP_V2_MAX_WINDOW) {
441 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
442 "the maximum body preread buffer size is %uz",
443 NGX_HTTP_V2_MAX_WINDOW);
444
445 return NGX_CONF_ERROR;
446 }
447
448 return NGX_CONF_OK;
449 }
450
451
452 static char *
422453 ngx_http_v2_streams_index_mask(ngx_conf_t *cf, void *post, void *data)
423454 {
424455 ngx_uint_t *np = data;
+1
-0
src/http/v2/ngx_http_v2_module.h less more
2424 ngx_uint_t concurrent_streams;
2525 size_t max_field_size;
2626 size_t max_header_size;
27 size_t preread_size;
2728 ngx_uint_t streams_index_mask;
2829 ngx_msec_t recv_timeout;
2930 ngx_msec_t idle_timeout;