http: Add core http compression code

This is the initial step to enabling HTTP compression on both static and
application responses.

This code itself doesn't do any actual compression, that will come in
subsequent commits. It just contains the core functions for initialising
structures that describe the available compressors and functions for
checking if compression should be done depending on various criteria.

Signed-off-by: Andrew Clayton <a.clayton@nginx.com>

2 files changed, 543 insertions, 0 deletions

diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c
new file mode 100644
index 00000000..f35afeff
--- /dev/null
+++ b/src/nxt_http_compression.c
@@ -0,0 +1,495 @@
+/*
+ * Copyright (C) Andrew Clayton
+ * Copyright (C) F5, Inc.
+ */
+
+#include <nxt_auto_config.h>
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+
+#include <nxt_router.h>
+#include <nxt_http.h>
+#include <nxt_tstr.h>
+#include <nxt_conf.h>
+#include <nxt_http_compression.h>
+
+
+#define NXT_COMP_LEVEL_UNSET               INT8_MIN
+
+
+typedef enum nxt_http_comp_scheme_e        nxt_http_comp_scheme_t;
+typedef struct nxt_http_comp_type_s        nxt_http_comp_type_t;
+typedef struct nxt_http_comp_opts_s        nxt_http_comp_opts_t;
+typedef struct nxt_http_comp_compressor_s  nxt_http_comp_compressor_t;
+typedef struct nxt_http_comp_ctx_s         nxt_http_comp_ctx_t;
+
+enum nxt_http_comp_scheme_e {
+    NXT_HTTP_COMP_SCHEME_IDENTITY = 0,
+
+    /* keep last */
+    NXT_HTTP_COMP_SCHEME_UNKNOWN
+};
+#define NXT_NR_COMPRESSORS  NXT_HTTP_COMP_SCHEME_UNKNOWN
+
+struct nxt_http_comp_type_s {
+    nxt_str_t                         token;
+    nxt_http_comp_scheme_t            scheme;
+    int8_t                            def_compr;
+    int8_t                            comp_min;
+    int8_t                            comp_max;
+
+    const nxt_http_comp_operations_t  *cops;
+};
+
+struct nxt_http_comp_opts_s {
+    int8_t                      level;
+    nxt_off_t                   min_len;
+};
+
+struct nxt_http_comp_compressor_s {
+    const nxt_http_comp_type_t  *type;
+    nxt_http_comp_opts_t        opts;
+};
+
+struct nxt_http_comp_ctx_s {
+    nxt_uint_t                      idx;
+
+    nxt_off_t                       resp_clen;
+
+    nxt_http_comp_compressor_ctx_t  ctx;
+};
+
+
+static nxt_tstr_t                  *nxt_http_comp_accept_encoding_query;
+static nxt_http_route_rule_t       *nxt_http_comp_mime_types_rule;
+static nxt_http_comp_compressor_t  *nxt_http_comp_enabled_compressors;
+static nxt_uint_t                  nxt_http_comp_nr_enabled_compressors;
+
+static nxt_thread_declare_data(nxt_http_comp_ctx_t,
+                               nxt_http_comp_compressor_ctx);
+
+#define nxt_http_comp_ctx()  nxt_thread_get_data(nxt_http_comp_compressor_ctx)
+
+static const nxt_conf_map_t  nxt_http_comp_compressors_opts_map[] = {
+    {
+        nxt_string("level"),
+        NXT_CONF_MAP_INT,
+        offsetof(nxt_http_comp_opts_t, level),
+    }, {
+        nxt_string("min_length"),
+        NXT_CONF_MAP_SIZE,
+        offsetof(nxt_http_comp_opts_t, min_len),
+    },
+};
+
+static const nxt_http_comp_type_t  nxt_http_comp_compressors[] = {
+    /* Keep this first */
+    {
+        .token      = nxt_string("identity"),
+        .scheme     = NXT_HTTP_COMP_SCHEME_IDENTITY,
+    },
+};
+
+
+bool
+nxt_http_comp_wants_compression(void)
+{
+    nxt_http_comp_ctx_t  *ctx = nxt_http_comp_ctx();
+
+    return ctx->idx;
+}
+
+
+static nxt_uint_t
+nxt_http_comp_compressor_lookup_enabled(const nxt_str_t *token)
+{
+    if (token->start[0] == '*') {
+        return NXT_HTTP_COMP_SCHEME_IDENTITY;
+    }
+
+    for (nxt_uint_t i = 0; i < nxt_http_comp_nr_enabled_compressors; i++) {
+        if (nxt_strstr_eq(token,
+                          &nxt_http_comp_enabled_compressors[i].type->token))
+        {
+            return i;
+        }
+    }
+
+    return NXT_HTTP_COMP_SCHEME_UNKNOWN;
+}
+
+
+/*
+ * We need to parse the 'Accept-Encoding` header as described by
+ * <https://www.rfc-editor.org/rfc/rfc9110.html#field.accept-encoding>
+ * which can take forms such as
+ *
+ *  Accept-Encoding: compress, gzip
+ *  Accept-Encoding:
+ *  Accept-Encoding: *
+ *  Accept-Encoding: compress;q=0.5, gzip;q=1.0
+ *  Accept-Encoding: gzip;q=1.0, identity;q=0.5, *;q=0
+ *
+ *  '*:q=0' means if the content being served has no 'Content-Coding'
+ *  matching an 'Accept-Encoding' entry then don't send any response.
+ *
+ * 'identity;q=0' seems to basically mean the same thing...
+ */
+static nxt_int_t
+nxt_http_comp_select_compressor(nxt_http_request_t *r, const nxt_str_t *token)
+{
+    bool       identity_allowed = true;
+    char       *str, *tkn, *tail, *cur;
+    double     weight = 0.0;
+    nxt_int_t  idx = NXT_HTTP_COMP_SCHEME_IDENTITY;
+
+    str = nxt_str_cstrz(r->mem_pool, token);
+    if (str == NULL) {
+        return NXT_HTTP_COMP_SCHEME_IDENTITY;
+    }
+
+    cur = tail = str;
+    /*
+     * To ease parsing the Accept-Encoding header, remove all spaces,
+     * which hold no semantic meaning.
+     */
+    for (; *cur != '\0'; cur++) {
+        if (*cur == ' ') {
+            continue;
+        }
+
+        *tail++ = *cur;
+    }
+    *tail = '\0';
+
+    while ((tkn = strsep(&str, ","))) {
+        char                    *qptr;
+        double                  qval = 1.0;
+        nxt_str_t               enc;
+        nxt_uint_t              ecidx;
+        nxt_http_comp_scheme_t  scheme;
+
+        qptr = strstr(tkn, ";q=");
+        if (qptr != NULL) {
+            nxt_errno = 0;
+
+            qval = strtod(qptr + 3, NULL);
+
+            if (nxt_errno == ERANGE || qval < 0.0 || qval > 1.0) {
+                continue;
+            }
+        }
+
+        enc.start = (u_char *)tkn;
+        enc.length = qptr != NULL ? (size_t)(qptr - tkn) : strlen(tkn);
+
+        ecidx = nxt_http_comp_compressor_lookup_enabled(&enc);
+        if (ecidx == NXT_HTTP_COMP_SCHEME_UNKNOWN) {
+            continue;
+        }
+
+        scheme = nxt_http_comp_enabled_compressors[ecidx].type->scheme;
+
+        if (qval == 0.0 && scheme == NXT_HTTP_COMP_SCHEME_IDENTITY) {
+            identity_allowed = false;
+        }
+
+        if (qval == 0.0 || qval < weight) {
+            continue;
+        }
+
+        idx = ecidx;
+        weight = qval;
+    }
+
+    if (idx == NXT_HTTP_COMP_SCHEME_IDENTITY && !identity_allowed) {
+        return -1;
+    }
+
+    return idx;
+}
+
+
+static nxt_int_t
+nxt_http_comp_set_header(nxt_http_request_t *r, nxt_uint_t comp_idx)
+{
+    const nxt_str_t   *token;
+    nxt_http_field_t  *f;
+
+    static const nxt_str_t  content_encoding_str =
+                                    nxt_string("Content-Encoding");
+
+    f = nxt_list_add(r->resp.fields);
+    if (nxt_slow_path(f == NULL)) {
+        return NXT_ERROR;
+    }
+
+    token = &nxt_http_comp_enabled_compressors[comp_idx].type->token;
+
+    *f = (nxt_http_field_t){};
+
+    f->name = content_encoding_str.start;
+    f->name_length = content_encoding_str.length;
+    f->value = token->start;
+    f->value_length = token->length;
+
+    return NXT_OK;
+}
+
+
+static bool
+nxt_http_comp_is_resp_content_encoded(const nxt_http_request_t *r)
+{
+    nxt_http_field_t  *f;
+
+    nxt_list_each(f, r->resp.fields) {
+        if (nxt_strcasecmp(f->name, (const u_char *)"Content-Encoding") == 0) {
+            return true;
+        }
+    } nxt_list_loop;
+
+    return false;
+}
+
+
+nxt_int_t
+nxt_http_comp_check_compression(nxt_task_t *task, nxt_http_request_t *r)
+{
+    int                         err;
+    nxt_int_t                   ret, idx;
+    nxt_off_t                   min_len;
+    nxt_str_t                   accept_encoding, mime_type = {};
+    nxt_router_conf_t           *rtcf;
+    nxt_http_comp_ctx_t         *ctx = nxt_http_comp_ctx();
+    nxt_http_comp_compressor_t  *compressor;
+
+    *ctx = (nxt_http_comp_ctx_t){ .resp_clen = -1 };
+
+    if (nxt_http_comp_nr_enabled_compressors == 0) {
+        return NXT_OK;
+    }
+
+    if (r->resp.content_length_n == 0) {
+        return NXT_OK;
+    }
+
+    if (r->resp.mime_type != NULL) {
+        mime_type = *r->resp.mime_type;
+    } else if (r->resp.content_type != NULL) {
+        mime_type.start = r->resp.content_type->value;
+        mime_type.length = r->resp.content_type->value_length;
+    }
+
+    if (mime_type.start == NULL) {
+        return NXT_OK;
+    }
+
+    if (nxt_http_comp_mime_types_rule != NULL) {
+        ret = nxt_http_route_test_rule(r, nxt_http_comp_mime_types_rule,
+                                       mime_type.start,
+                                       mime_type.length);
+        if (ret == 0) {
+            return NXT_OK;
+        }
+    }
+
+    rtcf = r->conf->socket_conf->router_conf;
+
+    if (nxt_http_comp_is_resp_content_encoded(r)) {
+        return NXT_OK;
+    }
+
+    ret = nxt_tstr_query_init(&r->tstr_query, rtcf->tstr_state, &r->tstr_cache,
+                              r, r->mem_pool);
+    if (nxt_slow_path(ret == NXT_ERROR)) {
+        return NXT_ERROR;
+    }
+
+    ret = nxt_tstr_query(task, r->tstr_query,
+                         nxt_http_comp_accept_encoding_query, &accept_encoding);
+    if (nxt_slow_path(ret != NXT_OK)) {
+        return NXT_ERROR;
+    }
+
+    idx = nxt_http_comp_select_compressor(r, &accept_encoding);
+    if (idx == -1) {
+        return NXT_HTTP_NOT_ACCEPTABLE;
+    }
+
+    if (idx == NXT_HTTP_COMP_SCHEME_IDENTITY) {
+        return NXT_OK;
+    }
+
+    compressor = &nxt_http_comp_enabled_compressors[idx];
+
+    if (r->resp.content_length_n > -1) {
+        ctx->resp_clen = r->resp.content_length_n;
+    } else if (r->resp.content_length != NULL) {
+        ctx->resp_clen =
+                strtol((char *)r->resp.content_length->value, NULL, 10);
+    }
+
+    min_len = compressor->opts.min_len;
+
+    if (ctx->resp_clen > -1 && ctx->resp_clen < min_len) {
+        return NXT_OK;
+    }
+
+    nxt_http_comp_set_header(r, idx);
+
+    ctx->idx = idx;
+    ctx->ctx.level = nxt_http_comp_enabled_compressors[idx].opts.level;
+
+    err = compressor->type->cops->init(&ctx->ctx);
+    if (nxt_slow_path(err)) {
+        return NXT_ERROR;
+    }
+
+    return NXT_OK;
+}
+
+
+static nxt_uint_t
+nxt_http_comp_compressor_token2idx(const nxt_str_t *token)
+{
+    for (nxt_uint_t i = 0; i < nxt_nitems(nxt_http_comp_compressors); i++) {
+        if (nxt_strstr_eq(token, &nxt_http_comp_compressors[i].token)) {
+            return i;
+        }
+    }
+
+    return NXT_HTTP_COMP_SCHEME_UNKNOWN;
+}
+
+
+bool
+nxt_http_comp_compressor_is_valid(const nxt_str_t *token)
+{
+    nxt_uint_t  idx;
+
+    idx = nxt_http_comp_compressor_token2idx(token);
+    if (idx != NXT_HTTP_COMP_SCHEME_UNKNOWN) {
+        return true;
+    }
+
+    return false;
+}
+
+
+static nxt_int_t
+nxt_http_comp_set_compressor(nxt_task_t *task, nxt_router_conf_t *rtcf,
+                             const nxt_conf_value_t *comp, nxt_uint_t index)
+{
+    nxt_int_t                   ret;
+    nxt_str_t                   token;
+    nxt_uint_t                  cidx;
+    nxt_conf_value_t            *obj;
+    nxt_http_comp_compressor_t  *compr;
+
+    static const nxt_str_t  token_str = nxt_string("encoding");
+
+    obj = nxt_conf_get_object_member(comp, &token_str, NULL);
+    if (obj == NULL) {
+        return NXT_ERROR;
+    }
+
+    nxt_conf_get_string(obj, &token);
+    cidx = nxt_http_comp_compressor_token2idx(&token);
+
+    compr = &nxt_http_comp_enabled_compressors[index];
+
+    compr->type = &nxt_http_comp_compressors[cidx];
+    compr->opts.level = compr->type->def_compr;
+    compr->opts.min_len = -1;
+
+    ret = nxt_conf_map_object(rtcf->mem_pool, comp,
+                              nxt_http_comp_compressors_opts_map,
+                              nxt_nitems(nxt_http_comp_compressors_opts_map),
+                              &compr->opts);
+    if (nxt_slow_path(ret == NXT_ERROR)) {
+        return NXT_ERROR;
+    }
+
+    if (compr->opts.level < compr->type->comp_min
+        || compr->opts.level > compr->type->comp_max)
+    {
+        nxt_log(task, NXT_LOG_NOTICE,
+                "Overriding invalid compression level for [%V] [%d] -> [%d]",
+                &compr->type->token, compr->opts.level,
+                compr->type->def_compr);
+        compr->opts.level = compr->type->def_compr;
+    }
+
+    return NXT_OK;
+}
+
+
+nxt_int_t
+nxt_http_comp_compression_init(nxt_task_t *task, nxt_router_conf_t *rtcf,
+                               const nxt_conf_value_t *comp_conf)
+{
+    nxt_int_t         ret;
+    nxt_uint_t        n = 1;  /* 'identity' */
+    nxt_conf_value_t  *comps, *mimes;
+
+    static const nxt_str_t  accept_enc_str =
+                                    nxt_string("$header_accept_encoding");
+    static const nxt_str_t  comps_str = nxt_string("compressors");
+    static const nxt_str_t  mimes_str = nxt_string("types");
+
+    mimes = nxt_conf_get_object_member(comp_conf, &mimes_str, NULL);
+    if (mimes != NULL) {
+        nxt_http_comp_mime_types_rule =
+                        nxt_http_route_types_rule_create(task,
+                                                         rtcf->mem_pool, mimes);
+        if (nxt_slow_path(nxt_http_comp_mime_types_rule == NULL)) {
+            return NXT_ERROR;
+        }
+    }
+
+    nxt_http_comp_accept_encoding_query =
+                            nxt_tstr_compile(rtcf->tstr_state, &accept_enc_str,
+                                             NXT_TSTR_STRZ);
+    if (nxt_slow_path(nxt_http_comp_accept_encoding_query == NULL)) {
+        return NXT_ERROR;
+    }
+
+    comps = nxt_conf_get_object_member(comp_conf, &comps_str, NULL);
+    if (nxt_slow_path(comps == NULL)) {
+        return NXT_ERROR;
+    }
+
+    if (nxt_conf_type(comps) == NXT_CONF_OBJECT) {
+        n++;
+    } else {
+        n += nxt_conf_object_members_count(comps);
+    }
+    nxt_http_comp_nr_enabled_compressors = n;
+
+    nxt_http_comp_enabled_compressors =
+                        nxt_mp_zalloc(rtcf->mem_pool,
+                                      sizeof(nxt_http_comp_compressor_t) * n);
+
+    nxt_http_comp_enabled_compressors[0] =
+        (nxt_http_comp_compressor_t){ .type = &nxt_http_comp_compressors[0],
+                                      .opts.level = NXT_COMP_LEVEL_UNSET,
+                                      .opts.min_len = -1 };
+
+    if (nxt_conf_type(comps) == NXT_CONF_OBJECT) {
+        return nxt_http_comp_set_compressor(task, rtcf, comps, 1);
+    }
+
+    for (nxt_uint_t i = 1; i < nxt_http_comp_nr_enabled_compressors; i++) {
+        nxt_conf_value_t  *obj;
+
+        obj = nxt_conf_get_array_element(comps, i - 1);
+        ret = nxt_http_comp_set_compressor(task, rtcf, obj, i);
+        if (ret == NXT_ERROR) {
+            return NXT_ERROR;
+        }
+    }
+
+    return NXT_OK;
+}
diff --git a/src/nxt_http_compression.h b/src/nxt_http_compression.h
new file mode 100644
index 00000000..4e63d34d
--- /dev/null
+++ b/src/nxt_http_compression.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) Andrew Clayton
+ * Copyright (C) F5, Inc.
+ */
+
+#ifndef _NXT_COMPRESSION_H_INCLUDED_
+#define _NXT_COMPRESSION_H_INCLUDED_
+
+#include <nxt_auto_config.h>
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+
+#include <nxt_main.h>
+#include <nxt_router.h>
+#include <nxt_string.h>
+#include <nxt_conf.h>
+
+
+typedef struct nxt_http_comp_compressor_ctx_s  nxt_http_comp_compressor_ctx_t;
+typedef struct nxt_http_comp_operations_s      nxt_http_comp_operations_t;
+
+struct nxt_http_comp_compressor_ctx_s {
+    int8_t level;
+
+    union {
+    };
+};
+
+struct nxt_http_comp_operations_s {
+    int      (*init)(nxt_http_comp_compressor_ctx_t *ctx);
+    size_t   (*bound)(const nxt_http_comp_compressor_ctx_t *ctx,
+                      size_t in_len);
+    ssize_t  (*deflate)(nxt_http_comp_compressor_ctx_t *ctx,
+                        const uint8_t *in_buf, size_t in_len,
+                        uint8_t *out_buf, size_t out_len, bool last);
+};
+
+
+extern bool nxt_http_comp_wants_compression(void);
+extern bool nxt_http_comp_compressor_is_valid(const nxt_str_t *token);
+extern nxt_int_t nxt_http_comp_check_compression(nxt_task_t *task,
+    nxt_http_request_t *r);
+extern nxt_int_t nxt_http_comp_compression_init(nxt_task_t *task,
+    nxt_router_conf_t *rtcf, const nxt_conf_value_t *comp_conf);
+
+#endif  /* _NXT_COMPRESSION_H_INCLUDED_ */