The "multipath" parameter of the "listen" directive.

When configured, it enables Multipath TCP support on a listen socket.
As of now it works on Linux starting with Linux 5.6 and glibc 2.32,
where it is enabled with an IPPROTO_MPTCP socket(2) protocol.

To avoid EADDRINUSE errors in bind() and listen() when transitioning
between sockets with different protocols, SO_REUSEPORT is set on both
sockets.  See f7f1607bf for potential implications.

Based on previous work by Maxime Dourov and Anthony Doeraene.

3 files changed, 47 insertions, 10 deletions

diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
index 7cae295eb..c269a97e6 100644
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -315,6 +315,25 @@ ngx_set_inherited_sockets(ngx_cycle_t *cycle)
             continue;
         }
 
+#ifdef SO_PROTOCOL
+
+        olen = sizeof(int);
+
+        if (getsockopt(ls[i].fd, SOL_SOCKET, SO_PROTOCOL,
+                       (void *) &ls[i].protocol, &olen)
+            == -1)
+        {
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno,
+                          "getsockopt(SO_PROTOCOL) %V failed, ignored",
+                          &ls[i].addr_text);
+            ls[i].protocol = 0;
+
+        } else if (ls[i].protocol == IPPROTO_TCP) {
+            ls[i].protocol = 0;
+        }
+
+#endif
+
 #if (NGX_HAVE_TCP_FASTOPEN)
 
         olen = sizeof(int);
@@ -436,12 +455,16 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
 
 #if (NGX_HAVE_REUSEPORT)
 
-            if (ls[i].add_reuseport) {
+            if (ls[i].add_reuseport || ls[i].change_protocol) {
 
                 /*
                  * to allow transition from a socket without SO_REUSEPORT
                  * to multiple sockets with SO_REUSEPORT, we have to set
                  * SO_REUSEPORT on the old socket before opening new ones
+                 *
+                 * to allow transition between different socket protocols
+                 * (e.g. IPPROTO_MPTCP), SO_REUSEPORT is set on both old
+                 * and new sockets
                  */
 
                 int  reuseport = 1;
@@ -474,7 +497,7 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
             }
 #endif
 
-            if (ls[i].fd != (ngx_socket_t) -1) {
+            if (ls[i].fd != (ngx_socket_t) -1 && !ls[i].change_protocol) {
                 continue;
             }
 
@@ -487,7 +510,8 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
                 continue;
             }
 
-            s = ngx_socket(ls[i].sockaddr->sa_family, ls[i].type, 0);
+            s = ngx_socket(ls[i].sockaddr->sa_family, ls[i].type,
+                           ls[i].protocol);
 
             if (s == (ngx_socket_t) -1) {
                 ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
@@ -517,7 +541,9 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
 
 #if (NGX_HAVE_REUSEPORT)
 
-            if (ls[i].reuseport && !ngx_test_config) {
+            if ((ls[i].reuseport || ls[i].change_protocol)
+                && !ngx_test_config)
+            {
                 int  reuseport;
 
                 reuseport = 1;
@@ -651,6 +677,7 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
 
             if (ls[i].type != SOCK_STREAM) {
                 ls[i].fd = s;
+                ls[i].open = 1;
                 continue;
             }
 
@@ -689,6 +716,7 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
             ls[i].listen = 1;
 
             ls[i].fd = s;
+            ls[i].open = 1;
         }
 
         if (!failed) {
diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
index 84dd80442..b7d236837 100644
--- a/src/core/ngx_connection.h
+++ b/src/core/ngx_connection.h
@@ -24,6 +24,7 @@ struct ngx_listening_s {
     ngx_str_t           addr_text;
 
     int                 type;
+    int                 protocol;
 
     int                 backlog;
     int                 rcvbuf;
@@ -75,6 +76,8 @@ struct ngx_listening_s {
     unsigned            keepalive:2;
     unsigned            quic:1;
 
+    unsigned            change_protocol:1;
+
     unsigned            deferred_accept:1;
     unsigned            delete_deferred:1;
     unsigned            add_deferred:1;
diff --git a/src/core/ngx_cycle.c b/src/core/ngx_cycle.c
index a75bdf878..e5fd40285 100644
--- a/src/core/ngx_cycle.c
+++ b/src/core/ngx_cycle.c
@@ -535,9 +535,15 @@ ngx_init_cycle(ngx_cycle_t *old_cycle)
                     == NGX_OK)
                 {
                     nls[n].fd = ls[i].fd;
-                    nls[n].inherited = ls[i].inherited;
                     nls[n].previous = &ls[i];
-                    ls[i].remain = 1;
+
+                    if (ls[i].protocol != nls[n].protocol) {
+                        nls[n].change_protocol = 1;
+
+                    } else {
+                        nls[n].inherited = ls[i].inherited;
+                        ls[i].remain = 1;
+                    }
 
                     if (ls[i].backlog != nls[n].backlog) {
                         nls[n].listen = 1;
@@ -590,7 +596,6 @@ ngx_init_cycle(ngx_cycle_t *old_cycle)
             }
 
             if (nls[n].fd == (ngx_socket_t) -1) {
-                nls[n].open = 1;
 #if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER)
                 if (nls[n].accept_filter) {
                     nls[n].add_deferred = 1;
@@ -605,20 +610,21 @@ ngx_init_cycle(ngx_cycle_t *old_cycle)
         }
 
     } else {
+#if (NGX_HAVE_DEFERRED_ACCEPT)
         ls = cycle->listening.elts;
         for (i = 0; i < cycle->listening.nelts; i++) {
-            ls[i].open = 1;
-#if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER)
+#ifdef SO_ACCEPTFILTER
             if (ls[i].accept_filter) {
                 ls[i].add_deferred = 1;
             }
 #endif
-#if (NGX_HAVE_DEFERRED_ACCEPT && defined TCP_DEFER_ACCEPT)
+#ifdef TCP_DEFER_ACCEPT
             if (ls[i].deferred_accept) {
                 ls[i].add_deferred = 1;
             }
 #endif
         }
+#endif
     }
 
     if (ngx_open_listening_sockets(cycle) != NGX_OK) {