QUIC: improved error handling in OpenSSL compat layer.

Previously ngx_quic_compat_create_record() could try to encrypt a TLS
record even if encryption context was missing, which resulted in a NULL
pointer dereference.

The context is created by ngx_quic_compat_set_encryption_secret() called
from the OpenSSL keylog callback.  If an error occurred in that function,
the context could remain missing.  This could happen under memory pressure,
if an allocation failed inside this function.

The fix is to handle errors from ngx_quic_compat_set_encryption_secret()
and set qc->error to trigger an error after SSL_do_handshake() return.
Also, a check for context is added to ngx_quic_compat_create_record()
to avoid other similar issues.

1 files changed, 10 insertions, 2 deletions

diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c
index 58298dcb8..c5f4a0f74 100644
--- a/src/event/quic/ngx_event_quic_openssl_compat.c
+++ b/src/event/quic/ngx_event_quic_openssl_compat.c
@@ -213,8 +213,12 @@ ngx_quic_compat_keylog_callback(const SSL *ssl, const char *line)
         com->method->set_read_secret((SSL *) ssl, level, cipher, secret, n);
         com->read_record = 0;
 
-        (void) ngx_quic_compat_set_encryption_secret(c, &com->keys, level,
-                                                     cipher, secret, n);
+        if (ngx_quic_compat_set_encryption_secret(c, &com->keys, level,
+                                                  cipher, secret, n)
+            != NGX_OK)
+        {
+            qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+        }
     }
 
     ngx_explicit_memzero(secret, n);
@@ -591,6 +595,10 @@ ngx_quic_compat_create_record(ngx_quic_compat_record_t *rec, ngx_str_t *res)
 
     secret = &rec->keys->secret;
 
+    if (secret->ctx == NULL) {
+        return NGX_ERROR;
+    }
+
     ngx_memcpy(nonce, secret->iv.data, secret->iv.len);
     ngx_quic_compute_nonce(nonce, sizeof(nonce), rec->number);