From 5cef7de7a116bab3af9097dac5a22f7652be4273 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 9 Jul 2020 16:21:37 +0300 Subject: Slice filter: clear original Accept-Ranges. The slice filter allows ranges for the response by setting the r->allow_ranges flag, which enables the range filter. If the range was not requested, the range filter adds an Accept-Ranges header to the response to signal the support for ranges. Previously, if an Accept-Ranges header was already present in the first slice response, client received two copies of this header. Now, the slice filter removes the Accept-Ranges header from the response prior to setting the r->allow_ranges flag. --- src/http/modules/ngx_http_slice_filter_module.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/http') diff --git a/src/http/modules/ngx_http_slice_filter_module.c b/src/http/modules/ngx_http_slice_filter_module.c index c1edbca2b..186380a2f 100644 --- a/src/http/modules/ngx_http_slice_filter_module.c +++ b/src/http/modules/ngx_http_slice_filter_module.c @@ -180,6 +180,11 @@ ngx_http_slice_header_filter(ngx_http_request_t *r) r->headers_out.content_range->hash = 0; r->headers_out.content_range = NULL; + if (r->headers_out.accept_ranges) { + r->headers_out.accept_ranges->hash = 0; + r->headers_out.accept_ranges = NULL; + } + r->allow_ranges = 1; r->subrequest_ranges = 1; r->single_range = 1; -- cgit From 4dd43dfca71f3fc2c6768606ff3700a4317a9176 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 22 Jul 2020 22:16:19 +0300 Subject: Xslt: disabled ranges. Previously, the document generated by the xslt filter was always fully sent to client even if a range was requested and response status was 206 with appropriate Content-Range. The xslt module is unable to serve a range because of suspending the header filter chain. By the moment full response xml is buffered by the xslt filter, range header filter is not called yet, but the range body filter has already been called and did nothing. The fix is to disable ranges by resetting the r->allow_ranges flag much like the image filter that employs a similar technique. --- src/http/modules/ngx_http_xslt_filter_module.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src/http') diff --git a/src/http/modules/ngx_http_xslt_filter_module.c b/src/http/modules/ngx_http_xslt_filter_module.c index b2f107dc0..8afd656af 100644 --- a/src/http/modules/ngx_http_xslt_filter_module.c +++ b/src/http/modules/ngx_http_xslt_filter_module.c @@ -233,6 +233,7 @@ ngx_http_xslt_header_filter(ngx_http_request_t *r) ngx_http_set_ctx(r, ctx, ngx_http_xslt_filter_module); r->main_filter_need_in_memory = 1; + r->allow_ranges = 0; return NGX_OK; } -- cgit From d2744ad26fef1e4f4f6e9c12e95b57866345c071 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Mon, 27 Jul 2020 16:02:15 +0300 Subject: FastCGI: fixed zero size buf alerts on extra data (ticket #2018). After 05e42236e95b (1.19.1) responses with extra data might result in zero size buffers being generated and "zero size buf" alerts in writer (if f->rest happened to be 0 when processing additional stdout data). --- src/http/modules/ngx_http_fastcgi_module.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) (limited to 'src/http') diff --git a/src/http/modules/ngx_http_fastcgi_module.c b/src/http/modules/ngx_http_fastcgi_module.c index e50d1a70d..5191880e3 100644 --- a/src/http/modules/ngx_http_fastcgi_module.c +++ b/src/http/modules/ngx_http_fastcgi_module.c @@ -2306,6 +2306,18 @@ ngx_http_fastcgi_input_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) break; } + if (f->rest == -2) { + f->rest = r->upstream->headers_in.content_length_n; + } + + if (f->rest == 0) { + ngx_log_error(NGX_LOG_WARN, p->log, 0, + "upstream sent more data than specified in " + "\"Content-Length\" header"); + p->upstream_done = 1; + break; + } + cl = ngx_chain_get_free_buf(p->pool, &p->free); if (cl == NULL) { return NGX_ERROR; @@ -2349,11 +2361,7 @@ ngx_http_fastcgi_input_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) b->last = f->last; } - if (f->rest == -2) { - f->rest = r->upstream->headers_in.content_length_n; - } - - if (f->rest >= 0) { + if (f->rest > 0) { if (b->last - b->pos > f->rest) { ngx_log_error(NGX_LOG_WARN, p->log, 0, @@ -2564,6 +2572,14 @@ ngx_http_fastcgi_non_buffered_filter(void *data, ssize_t bytes) break; } + if (f->rest == 0) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "upstream sent more data than specified in " + "\"Content-Length\" header"); + u->length = 0; + break; + } + cl = ngx_chain_get_free_buf(r->pool, &u->free_bufs); if (cl == NULL) { return NGX_ERROR; @@ -2594,7 +2610,7 @@ ngx_http_fastcgi_non_buffered_filter(void *data, ssize_t bytes) b->last = f->last; } - if (f->rest >= 0) { + if (f->rest > 0) { if (b->last - b->pos > f->rest) { ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, -- cgit From bd7dad5b0eb9f667a9c66ea5175a017ac51cd027 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Thu, 6 Aug 2020 05:02:22 +0300 Subject: Added size check to ngx_http_alloc_large_header_buffer(). This ensures that copying won't write more than the buffer size even if the buffer comes from hc->free and it is smaller than the large client header buffer size in the virtual host configuration. This might happen if size of large client header buffers is different in name-based virtual hosts, similarly to the problem with number of buffers fixed in 6926:e662cbf1b932. --- src/http/ngx_http_request.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/http') diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c index 6feb6cc31..257c4064b 100644 --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -1647,6 +1647,12 @@ ngx_http_alloc_large_header_buffer(ngx_http_request_t *r, ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http large header copy: %uz", r->header_in->pos - old); + if (r->header_in->pos - old > b->end - b->start) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "too large header to copy"); + return NGX_ERROR; + } + new = b->start; ngx_memcpy(new, old, r->header_in->pos - old); -- cgit From 9edc93fe0ed60bac336d11f7d20d3c2ed9db3227 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Thu, 6 Aug 2020 05:02:44 +0300 Subject: Request body: all read data are now sent to filters. This is a prerequisite for the next change to allow large reads on chunk boundaries. --- src/http/ngx_http_request_body.c | 53 ++++++++-------------------------------- 1 file changed, 10 insertions(+), 43 deletions(-) (limited to 'src/http') diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c index c4f092e59..d21c45863 100644 --- a/src/http/ngx_http_request_body.c +++ b/src/http/ngx_http_request_body.c @@ -282,28 +282,12 @@ ngx_http_do_read_client_request_body(ngx_http_request_t *r) for ( ;; ) { if (rb->buf->last == rb->buf->end) { - if (rb->buf->pos != rb->buf->last) { + /* update chains */ - /* pass buffer to request body filter chain */ + rc = ngx_http_request_body_filter(r, NULL); - out.buf = rb->buf; - out.next = NULL; - - rc = ngx_http_request_body_filter(r, &out); - - if (rc != NGX_OK) { - return rc; - } - - } else { - - /* update chains */ - - rc = ngx_http_request_body_filter(r, NULL); - - if (rc != NGX_OK) { - return rc; - } + if (rc != NGX_OK) { + return rc; } if (rb->busy != NULL) { @@ -355,17 +339,15 @@ ngx_http_do_read_client_request_body(ngx_http_request_t *r) rb->buf->last += n; r->request_length += n; - if (n == rest) { - /* pass buffer to request body filter chain */ + /* pass buffer to request body filter chain */ - out.buf = rb->buf; - out.next = NULL; + out.buf = rb->buf; + out.next = NULL; - rc = ngx_http_request_body_filter(r, &out); + rc = ngx_http_request_body_filter(r, &out); - if (rc != NGX_OK) { - return rc; - } + if (rc != NGX_OK) { + return rc; } if (rb->rest == 0) { @@ -386,21 +368,6 @@ ngx_http_do_read_client_request_body(ngx_http_request_t *r) if (!c->read->ready) { - if (r->request_body_no_buffering - && rb->buf->pos != rb->buf->last) - { - /* pass buffer to request body filter chain */ - - out.buf = rb->buf; - out.next = NULL; - - rc = ngx_http_request_body_filter(r, &out); - - if (rc != NGX_OK) { - return rc; - } - } - clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); ngx_add_timer(c->read, clcf->client_body_timeout); -- cgit From 150cbb017b4fda599dcda172dca87ca11f6219f1 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Thu, 6 Aug 2020 05:02:55 +0300 Subject: Request body: allowed large reads on chunk boundaries. If some additional data from a pipelined request happens to be read into the body buffer, we copy it to r->header_in or allocate an additional large client header buffer for it. --- src/http/ngx_http_request_body.c | 121 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 113 insertions(+), 8 deletions(-) (limited to 'src/http') diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c index d21c45863..106ac3dc6 100644 --- a/src/http/ngx_http_request_body.c +++ b/src/http/ngx_http_request_body.c @@ -12,6 +12,8 @@ static void ngx_http_read_client_request_body_handler(ngx_http_request_t *r); static ngx_int_t ngx_http_do_read_client_request_body(ngx_http_request_t *r); +static ngx_int_t ngx_http_copy_pipelined_header(ngx_http_request_t *r, + ngx_buf_t *buf); static ngx_int_t ngx_http_write_request_body(ngx_http_request_t *r); static ngx_int_t ngx_http_read_discarded_request_body(ngx_http_request_t *r); static ngx_int_t ngx_http_discard_request_body_filter(ngx_http_request_t *r, @@ -379,6 +381,10 @@ ngx_http_do_read_client_request_body(ngx_http_request_t *r) } } + if (ngx_http_copy_pipelined_header(r, rb->buf) != NGX_OK) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + if (c->read->timer_set) { ngx_del_timer(c->read); } @@ -392,6 +398,88 @@ ngx_http_do_read_client_request_body(ngx_http_request_t *r) } +static ngx_int_t +ngx_http_copy_pipelined_header(ngx_http_request_t *r, ngx_buf_t *buf) +{ + size_t n; + ngx_buf_t *b; + ngx_chain_t *cl; + ngx_http_connection_t *hc; + ngx_http_core_srv_conf_t *cscf; + + b = r->header_in; + n = buf->last - buf->pos; + + if (buf == b || n == 0) { + return NGX_OK; + } + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http body pipelined header: %uz", n); + + /* + * if there is a pipelined request in the client body buffer, + * copy it to the r->header_in buffer if there is enough room, + * or allocate a large client header buffer + */ + + if (n > (size_t) (b->end - b->last)) { + + hc = r->http_connection; + + if (hc->free) { + cl = hc->free; + hc->free = cl->next; + + b = cl->buf; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http large header free: %p %uz", + b->pos, b->end - b->last); + + } else { + cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); + + b = ngx_create_temp_buf(r->connection->pool, + cscf->large_client_header_buffers.size); + if (b == NULL) { + return NGX_ERROR; + } + + cl = ngx_alloc_chain_link(r->connection->pool); + if (cl == NULL) { + return NGX_ERROR; + } + + cl->buf = b; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http large header alloc: %p %uz", + b->pos, b->end - b->last); + } + + cl->next = hc->busy; + hc->busy = cl; + hc->nbusy++; + + r->header_in = b; + + if (n > (size_t) (b->end - b->last)) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "too large pipelined header after reading body"); + return NGX_ERROR; + } + } + + ngx_memcpy(b->last, buf->pos, n); + + b->last += n; + r->request_length -= n; + + return NGX_OK; +} + + static ngx_int_t ngx_http_write_request_body(ngx_http_request_t *r) { @@ -637,8 +725,7 @@ ngx_http_read_discarded_request_body(ngx_http_request_t *r) for ( ;; ) { if (r->headers_in.content_length_n == 0) { - r->read_event_handler = ngx_http_block_reading; - return NGX_OK; + break; } if (!r->connection->read->ready) { @@ -672,15 +759,24 @@ ngx_http_read_discarded_request_body(ngx_http_request_t *r) return rc; } } + + if (ngx_http_copy_pipelined_header(r, &b) != NGX_OK) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + r->read_event_handler = ngx_http_block_reading; + + return NGX_OK; } static ngx_int_t ngx_http_discard_request_body_filter(ngx_http_request_t *r, ngx_buf_t *b) { - size_t size; - ngx_int_t rc; - ngx_http_request_body_t *rb; + size_t size; + ngx_int_t rc; + ngx_http_request_body_t *rb; + ngx_http_core_srv_conf_t *cscf; if (r->headers_in.chunked) { @@ -735,7 +831,10 @@ ngx_http_discard_request_body_filter(ngx_http_request_t *r, ngx_buf_t *b) /* set amount of data we want to see next time */ - r->headers_in.content_length_n = rb->chunked->length; + cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); + + r->headers_in.content_length_n = ngx_max(rb->chunked->length, + (off_t) cscf->large_client_header_buffers.size); break; } @@ -903,6 +1002,7 @@ ngx_http_request_body_chunked_filter(ngx_http_request_t *r, ngx_chain_t *in) ngx_chain_t *cl, *out, *tl, **ll; ngx_http_request_body_t *rb; ngx_http_core_loc_conf_t *clcf; + ngx_http_core_srv_conf_t *cscf; rb = r->request_body; @@ -916,8 +1016,10 @@ ngx_http_request_body_chunked_filter(ngx_http_request_t *r, ngx_chain_t *in) return NGX_HTTP_INTERNAL_SERVER_ERROR; } + cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); + r->headers_in.content_length_n = 0; - rb->rest = 3; + rb->rest = cscf->large_client_header_buffers.size; } out = NULL; @@ -1024,7 +1126,10 @@ ngx_http_request_body_chunked_filter(ngx_http_request_t *r, ngx_chain_t *in) /* set rb->rest, amount of data we want to see next time */ - rb->rest = rb->chunked->length; + cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); + + rb->rest = ngx_max(rb->chunked->length, + (off_t) cscf->large_client_header_buffers.size); break; } -- cgit From 130a5e71269200154b55e85d9e30186feaeb64a7 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Thu, 6 Aug 2020 05:02:57 +0300 Subject: Request body: optimized handling of small chunks. If there is a previous buffer, copy small chunks into it instead of allocating additional buffer. --- src/http/ngx_http_request_body.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'src/http') diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c index 106ac3dc6..71d7e9ab8 100644 --- a/src/http/ngx_http_request_body.c +++ b/src/http/ngx_http_request_body.c @@ -1027,6 +1027,8 @@ ngx_http_request_body_chunked_filter(ngx_http_request_t *r, ngx_chain_t *in) for (cl = in; cl; cl = cl->next) { + b = NULL; + for ( ;; ) { ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, @@ -1061,6 +1063,29 @@ ngx_http_request_body_chunked_filter(ngx_http_request_t *r, ngx_chain_t *in) return NGX_HTTP_REQUEST_ENTITY_TOO_LARGE; } + if (b + && rb->chunked->size <= 128 + && cl->buf->last - cl->buf->pos >= rb->chunked->size) + { + r->headers_in.content_length_n += rb->chunked->size; + + if (rb->chunked->size < 8) { + + while (rb->chunked->size) { + *b->last++ = *cl->buf->pos++; + rb->chunked->size--; + } + + } else { + ngx_memmove(b->last, cl->buf->pos, rb->chunked->size); + b->last += rb->chunked->size; + cl->buf->pos += rb->chunked->size; + rb->chunked->size = 0; + } + + continue; + } + tl = ngx_chain_get_free_buf(r->pool, &rb->free); if (tl == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; -- cgit From 1d696cd37947ef816bde4d54d7b6f97374f1151d Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Mon, 10 Aug 2020 18:52:20 +0300 Subject: HTTP/2: fixed c->timedout flag on timed out connections. Without the flag, SSL shutdown is attempted on such connections, resulting in useless work and/or bogus "SSL_shutdown() failed (SSL: ... bad write retry)" critical log messages if there are blocked writes. --- src/http/v2/ngx_http_v2.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src/http') diff --git a/src/http/v2/ngx_http_v2.c b/src/http/v2/ngx_http_v2.c index ec553ecfe..51c8b0a71 100644 --- a/src/http/v2/ngx_http_v2.c +++ b/src/http/v2/ngx_http_v2.c @@ -475,6 +475,7 @@ ngx_http_v2_write_handler(ngx_event_t *wev) ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http2 write event timed out"); c->error = 1; + c->timedout = 1; ngx_http_v2_finalize_connection(h2c, 0); return; } -- cgit From eae2b2fdf15c52f058c0c08763a5c373997d0535 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Mon, 10 Aug 2020 18:52:34 +0300 Subject: SSL: disabled sending shutdown after ngx_http_test_reading(). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sending shutdown when ngx_http_test_reading() detects the connection is closed can result in "SSL_shutdown() failed (SSL: ... bad write retry)" critical log messages if there are blocked writes. Fix is to avoid sending shutdown via the c->ssl->no_send_shutdown flag, similarly to how it is done in ngx_http_keepalive_handler() for kqueue when pending EOF is detected. Reported by Jan Prachař (http://mailman.nginx.org/pipermail/nginx-devel/2018-December/011702.html). --- src/http/ngx_http_request.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/http') diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c index 257c4064b..f80785d8f 100644 --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -2992,6 +2992,12 @@ closed: rev->error = 1; } +#if (NGX_HTTP_SSL) + if (c->ssl) { + c->ssl->no_send_shutdown = 1; + } +#endif + ngx_log_error(NGX_LOG_INFO, c->log, err, "client prematurely closed connection"); -- cgit