Added primitive flow control mechanisms.

 + MAX_STREAM_DATA frame is sent when recv() is performed on stream
   The new value is a sum of total bytes received by stream + free
   space in a buffer;

   The sending of MAX_STREM_DATA frame in response to STREAM_DATA_BLOCKED
   frame is adjusted to follow the same logic as above.

 + MAX_DATA frame is sent when total amount of received data is 2x
   of current limit.  The limit is doubled.

 + Default values of transport parameters are adjusted to more meaningful
   values:

   initial stream limits are set to quic buffer size instead of
   unrealistically small 255.

   initial max data is decreased to 16 buffer sizes, in an assumption that
   this is enough for a relatively short connection, instead of randomly
   chosen big number.

All this allows to initiate a stable flow of streams that does not block
on stream/connection limits (tested with FF 77.0a1 and 100K requests)

3 files changed, 92 insertions, 9 deletions

diff --git a/src/event/ngx_event_quic.c b/src/event/ngx_event_quic.c
index 1373e0cf7..db28cd548 100644
--- a/src/event/ngx_event_quic.c
+++ b/src/event/ngx_event_quic.c
@@ -47,6 +47,9 @@ typedef struct {
     ngx_connection_handler_pt         handler;
 
     ngx_uint_t                        id_counter;
+
+    uint64_t                          total_received;
+    uint64_t                          max_data;
 } ngx_quic_streams_t;
 
 
@@ -535,6 +538,8 @@ ngx_quic_new_connection(ngx_connection_t *c, ngx_ssl_t *ssl, ngx_quic_tp_t *tp,
     ctp->ack_delay_exponent = NGX_QUIC_DEFAULT_ACK_DELAY_EXPONENT;
     ctp->max_ack_delay = NGX_QUIC_DEFAULT_MAX_ACK_DELAY;
 
+    qc->streams.max_data = qc->tp.initial_max_data;
+
     qc->dcid.len = pkt->dcid.len;
     qc->dcid.data = ngx_pnalloc(c->pool, pkt->dcid.len);
     if (qc->dcid.data == NULL) {
@@ -1963,7 +1968,7 @@ ngx_quic_handle_stream_data_blocked_frame(ngx_connection_t *c,
     }
 
     b = sn->b;
-    n = (b->pos - b->start) + (b->end - b->last);
+    n = sn->fs.received + (b->pos - b->start) + (b->end - b->last);
 
     frame = ngx_quic_alloc_frame(c, 0);
     if (frame == NULL) {
@@ -2559,13 +2564,18 @@ ngx_quic_create_stream(ngx_connection_t *c, uint64_t id, size_t rcvbuf_size)
 static ssize_t
 ngx_quic_stream_recv(ngx_connection_t *c, u_char *buf, size_t size)
 {
-    ssize_t             len;
-    ngx_buf_t          *b;
-    ngx_event_t        *rev;
-    ngx_quic_stream_t  *qs;
+    ssize_t                 len;
+    ngx_buf_t              *b;
+    ngx_event_t            *rev;
+    ngx_connection_t       *pc;
+    ngx_quic_frame_t       *frame;
+    ngx_quic_stream_t      *qs;
+    ngx_quic_connection_t  *qc;
 
     qs = c->qs;
     b = qs->b;
+    pc = qs->parent;
+    qc = pc->quic;
     rev = c->read;
 
     ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
@@ -2589,6 +2599,7 @@ ngx_quic_stream_recv(ngx_connection_t *c, u_char *buf, size_t size)
     ngx_memcpy(buf, b->pos, len);
 
     b->pos += len;
+    qc->streams.total_received += len;
 
     if (b->pos == b->last) {
         b->pos = b->start;
@@ -2599,6 +2610,50 @@ ngx_quic_stream_recv(ngx_connection_t *c, u_char *buf, size_t size)
     ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
                    "quic recv: %z of %uz", len, size);
 
+    if (!rev->pending_eof) {
+        frame = ngx_quic_alloc_frame(pc, 0);
+        if (frame == NULL) {
+            return NGX_ERROR;
+        }
+
+        frame->level = ssl_encryption_application;
+        frame->type = NGX_QUIC_FT_MAX_STREAM_DATA;
+        frame->u.max_stream_data.id = qs->id;
+        frame->u.max_stream_data.limit = qs->fs.received + (b->pos - b->start)
+                                         + (b->end - b->last);
+
+        ngx_sprintf(frame->info, "MAX_STREAM_DATA id:%d limit:%d l=%d on recv",
+                    (int) frame->u.max_stream_data.id,
+                    (int) frame->u.max_stream_data.limit,
+                    frame->level);
+
+        ngx_quic_queue_frame(pc->quic, frame);
+    }
+
+    if ((qc->streams.max_data / 2) < qc->streams.total_received) {
+
+        frame = ngx_quic_alloc_frame(pc, 0);
+
+        if (frame == NULL) {
+            return NGX_ERROR;
+        }
+
+        qc->streams.max_data *= 2;
+
+        frame->level = ssl_encryption_application;
+        frame->type = NGX_QUIC_FT_MAX_DATA;
+        frame->u.max_data.max_data = qc->streams.max_data;
+
+        ngx_sprintf(frame->info, "MAX_DATA max_data:%d level=%d on recv",
+                    (int) frame->u.max_data.max_data, frame->level);
+
+        ngx_quic_queue_frame(pc->quic, frame);
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "quic recv: increased max data: %ui",
+                       qc->streams.max_data);
+    }
+
     return len;
 }
 
diff --git a/src/event/ngx_event_quic_transport.c b/src/event/ngx_event_quic_transport.c
index 1e5a7ea8f..5dc0f1d72 100644
--- a/src/event/ngx_event_quic_transport.c
+++ b/src/event/ngx_event_quic_transport.c
@@ -77,6 +77,8 @@ static size_t ngx_quic_create_max_streams(u_char *p,
     ngx_quic_max_streams_frame_t *ms);
 static size_t ngx_quic_create_max_stream_data(u_char *p,
     ngx_quic_max_stream_data_frame_t *ms);
+static size_t ngx_quic_create_max_data(u_char *p,
+    ngx_quic_max_data_frame_t *md);
 static size_t ngx_quic_create_close(u_char *p, ngx_quic_close_frame_t *cl);
 
 static ngx_int_t ngx_quic_parse_transport_param(u_char *p, u_char *end,
@@ -1196,6 +1198,9 @@ ngx_quic_create_frame(u_char *p, ngx_quic_frame_t *f)
     case NGX_QUIC_FT_MAX_STREAM_DATA:
         return ngx_quic_create_max_stream_data(p, &f->u.max_stream_data);
 
+    case NGX_QUIC_FT_MAX_DATA:
+        return ngx_quic_create_max_data(p, &f->u.max_data);
+
     default:
         /* BUG: unsupported frame type generated */
         return NGX_ERROR;
@@ -1616,6 +1621,27 @@ ngx_quic_create_max_stream_data(u_char *p, ngx_quic_max_stream_data_frame_t *ms)
 }
 
 
+static size_t
+ngx_quic_create_max_data(u_char *p, ngx_quic_max_data_frame_t *md)
+{
+    size_t   len;
+    u_char  *start;
+
+    if (p == NULL) {
+        len = ngx_quic_varint_len(NGX_QUIC_FT_MAX_DATA);
+        len += ngx_quic_varint_len(md->max_data);
+        return len;
+    }
+
+    start = p;
+
+    ngx_quic_build_int(&p, NGX_QUIC_FT_MAX_DATA);
+    ngx_quic_build_int(&p, md->max_data);
+
+    return p - start;
+}
+
+
 ssize_t
 ngx_quic_create_transport_params(u_char *pos, u_char *end, ngx_quic_tp_t *tp)
 {
diff --git a/src/http/v3/ngx_http_v3_module.c b/src/http/v3/ngx_http_v3_module.c
index 279210337..4306206c9 100644
--- a/src/http/v3/ngx_http_v3_module.c
+++ b/src/http/v3/ngx_http_v3_module.c
@@ -266,18 +266,20 @@ ngx_http_v3_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child)
                               NGX_QUIC_DEFAULT_MAX_PACKET_SIZE);
 
     ngx_conf_merge_uint_value(conf->quic.initial_max_data,
-                              prev->quic.initial_max_data, 10000000);
+                              prev->quic.initial_max_data,
+                              16 * NGX_QUIC_STREAM_BUFSIZE);
 
     ngx_conf_merge_uint_value(conf->quic.initial_max_stream_data_bidi_local,
                               prev->quic.initial_max_stream_data_bidi_local,
-                              255);
+                              NGX_QUIC_STREAM_BUFSIZE);
 
     ngx_conf_merge_uint_value(conf->quic.initial_max_stream_data_bidi_remote,
                               prev->quic.initial_max_stream_data_bidi_remote,
-                              255);
+                              NGX_QUIC_STREAM_BUFSIZE);
 
     ngx_conf_merge_uint_value(conf->quic.initial_max_stream_data_uni,
-                              prev->quic.initial_max_stream_data_uni, 255);
+                              prev->quic.initial_max_stream_data_uni,
+                              NGX_QUIC_STREAM_BUFSIZE);
 
     ngx_conf_merge_uint_value(conf->quic.initial_max_streams_bidi,
                               prev->quic.initial_max_streams_bidi, 16);