From a560cbf992724eb0195544729d67286fd81f4bac Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Fri, 30 Dec 2022 00:07:20 +0000 Subject: Python: Do some cleanup in nxt_python3_init_config(). This is a preparatory patch for future work and cleans up the code a little in the Python 3.8+ variant of nxt_python3_init_config(). The main advantage being we no longer have calls to PyConfig_Clear() in two different paths. The variables have a little extra space in their declarations to allow for the next patch which introduces a variable with a longer type name, which will help reduce the size of the diff. Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/python/nxt_python.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/python/nxt_python.c b/src/python/nxt_python.c index bdb04579..83f2544e 100644 --- a/src/python/nxt_python.c +++ b/src/python/nxt_python.c @@ -75,8 +75,11 @@ static nxt_python_proto_t nxt_py_proto; static nxt_int_t nxt_python3_init_config(nxt_int_t pep405) { - PyStatus status; - PyConfig config; + PyConfig config; + PyStatus status; + nxt_int_t ret; + + ret = NXT_ERROR; PyConfig_InitIsolatedConfig(&config); @@ -84,29 +87,28 @@ nxt_python3_init_config(nxt_int_t pep405) status = PyConfig_SetString(&config, &config.program_name, nxt_py_home); if (PyStatus_Exception(status)) { - goto pyinit_exception; + goto out_config_clear; } } else { - status =PyConfig_SetString(&config, &config.home, nxt_py_home); + status = PyConfig_SetString(&config, &config.home, nxt_py_home); if (PyStatus_Exception(status)) { - goto pyinit_exception; + goto out_config_clear; } } status = Py_InitializeFromConfig(&config); if (PyStatus_Exception(status)) { - goto pyinit_exception; + goto out_config_clear; } - PyConfig_Clear(&config); - return NXT_OK; + ret = NXT_OK; -pyinit_exception: +out_config_clear: PyConfig_Clear(&config); - return NXT_ERROR; + return ret; } #elif PY_MAJOR_VERSION == 3 -- cgit From f3d05bba525c048d8ca905831a7729d7f90c94bc Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Thu, 5 Jan 2023 22:04:32 +0000 Subject: Python: Fix enabling of UTF-8 in some situations. There was a couple of reports of Python applications failing due to the following type of error File "/opt/netbox/netbox/netbox/configuration.py", line 25, in _import print(f"\U0001f9ec loaded config '{path}'") UnicodeEncodeError: 'ascii' codec can't encode character '\U0001f9ec' in position 0: ordinal not in range(128) due to the use of Unicode text in the print() statement. This only happened for python 3.8+ when using the "home" configuration option as this meant we were going through the new PyConfig configuration. When using this new configuration method with the 'isolated' specific API (for embedded Python) UTF-8 is disabled by default, PyPreConfig->utf8_mode = 0. To fix this we need to setup the Python pre config and enable utf-8 mode. However rather than enable utf-8 unconditionally we can set to it to -1 so that it will use the LC_CTYPE environment variable to determine whether to enable utf-8 mode or not. utf-8 mode will be enabled if LC_CTYPE is either: C, POSIX or some specific UTF-8 locale. This is the default utf8_mode setting when using the non-isolated PyPreConfig API. Reported-by: Tobias Genannt Tested-by: Tobias Genannt Link: Link: Fixes: 491d0f70 ("Python: Added support for Python 3.11.") Closes: Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/python/nxt_python.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'src') diff --git a/src/python/nxt_python.c b/src/python/nxt_python.c index 83f2544e..7c059649 100644 --- a/src/python/nxt_python.c +++ b/src/python/nxt_python.c @@ -78,9 +78,23 @@ nxt_python3_init_config(nxt_int_t pep405) PyConfig config; PyStatus status; nxt_int_t ret; + PyPreConfig preconfig; ret = NXT_ERROR; + PyPreConfig_InitIsolatedConfig(&preconfig); + /* + * Determine whether to use UTF-8 mode or not, UTF-8 + * will be enabled if LC_CTYPE is C, POSIX or some + * specific UTF-8 locale. + */ + preconfig.utf8_mode = -1; + + status = Py_PreInitialize(&preconfig); + if (PyStatus_Exception(status)) { + return ret; + } + PyConfig_InitIsolatedConfig(&config); if (pep405) { -- cgit From 2c7e1bb92f6a3091b8fab872b93fa4791fbe14c8 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Fri, 2 Dec 2022 17:58:20 +0000 Subject: Fix endianness detection in nxt_websocket_header_t. The nxt_websocket_header_t structure defines the layout of a websocket frame header. As the websocket frame is mapped directly onto this structure its layout needs to match how it's coming off the network. The network being big endian means on big endian systems the structure layout can simply match that of the websocket frame header. On little endian systems we need to reverse the two bytes. This was done via the BYTE_ORDER, BIG_ENDIAN and LITTLE_ENDIAN macros, however these are not universal, e.g they are _not_ defined on illumos (OpenSolaris / OpenIndiana) and so we get the following compiler errors In file included from src/nxt_h1proto.c:12:0: src/nxt_websocket_header.h:25:13: error: duplicate member 'opcode' uint8_t opcode:4; ^~~~~~ src/nxt_websocket_header.h:26:13: error: duplicate member 'rsv3' uint8_t rsv3:1; ^~~~ src/nxt_websocket_header.h:27:13: error: duplicate member 'rsv2' uint8_t rsv2:1; ^~~~ src/nxt_websocket_header.h:28:13: error: duplicate member 'rsv1' uint8_t rsv1:1; ^~~~ src/nxt_websocket_header.h:29:13: error: duplicate member 'fin' uint8_t fin:1; ^~~ src/nxt_websocket_header.h:31:13: error: duplicate member 'payload_len' uint8_t payload_len:7; ^~~~~~~~~~~ src/nxt_websocket_header.h:32:13: error: duplicate member 'mask' uint8_t mask:1; ^~~~ This commit fixes that by using the new NXT_HAVE_{BIG,LITTLE}_ENDIAN macros introduced in the previous commit. Closes: Fixes: e501c74 ("Introducing websocket support in router and libunit.") Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_websocket_header.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/nxt_websocket_header.h b/src/nxt_websocket_header.h index f75dfacd..cb7431dd 100644 --- a/src/nxt_websocket_header.h +++ b/src/nxt_websocket_header.h @@ -10,7 +10,7 @@ typedef struct { -#if (BYTE_ORDER == BIG_ENDIAN) +#if (NXT_HAVE_BIG_ENDIAN) uint8_t fin:1; uint8_t rsv1:1; uint8_t rsv2:1; @@ -21,7 +21,7 @@ typedef struct { uint8_t payload_len:7; #endif -#if (BYTE_ORDER == LITTLE_ENDIAN) +#if (NXT_HAVE_LITTLE_ENDIAN) uint8_t opcode:4; uint8_t rsv3:1; uint8_t rsv2:1; -- cgit From 97caab0e7a40c4034888e3269cdcbb858e47b45b Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Mon, 7 Nov 2022 00:06:43 +0000 Subject: PHP: Fix a potential problem parsing the path. @dward on GitHub reported an issue with a URL like http://foo.bar/test.php?blah=test.php/foo where we would end up trying to run the script test.php?blah=test.php In the PHP module the format 'file.php/' is treated as a special case in nxt_php_dynamic_request() where we check the _path_ part of the url for the string '.php/'. The problem is that the path actually also contains the query string, thus we were finding 'test.php/' in the above URL and treating that whole path as the script to run. The fix is simple, replace the strstr(3) with a memmem(3), where we can limit the amount of path we use for the check. The trick here and what is not obvious from the code is that while path.start points to the whole path including the query string, path.length only contains the length of the _path_ part. NOTE: memmem(3) is a GNU extension and is neither specified by POSIX or ISO C, however it is available on a number of other systems, including: FreeBSD, OpenBSD, NetBSD, illumos, and macOS. If it comes to it we can implement a simple alternative for systems which lack memmem(3). This also adds a test case (provided by @dward) to cover this. Closes: Cc: Andrei Zeliankou Reviewed-by: Alejandro Colomar Reviewed-by: Andrei Zeliankou [test] Signed-off-by: Andrew Clayton --- src/nxt_php_sapi.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 126a4684..d2494938 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -1025,7 +1025,8 @@ nxt_php_dynamic_request(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) nxt_str_null(&script_name); - ctx->path_info.start = (u_char *) strstr((char *) path.start, ".php/"); + ctx->path_info.start = memmem(path.start, path.length, ".php/", + strlen(".php/")); if (ctx->path_info.start != NULL) { ctx->path_info.start += 4; path.length = ctx->path_info.start - path.start; -- cgit From 141deec353c546674719d2bffe2fadeb174c4415 Mon Sep 17 00:00:00 2001 From: Zhidao HONG Date: Tue, 17 Jan 2023 10:37:45 +0800 Subject: NJS: added the keys API for the request objects. This commit is to loop through the request objects headers, arguments, and cookies. --- src/nxt_http_js.c | 133 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 100 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/src/nxt_http_js.c b/src/nxt_http_js.c index 5a08a309..72ba761f 100644 --- a/src/nxt_http_js.c +++ b/src/nxt_http_js.c @@ -15,15 +15,19 @@ static njs_int_t nxt_http_js_ext_host(njs_vm_t *vm, njs_object_prop_t *prop, static njs_int_t nxt_http_js_ext_remote_addr(njs_vm_t *vm, njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval, njs_value_t *retval); -static njs_int_t nxt_http_js_ext_get_arg(njs_vm_t *vm, +static njs_int_t nxt_http_js_ext_get_args(njs_vm_t *vm, njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval, njs_value_t *retval); static njs_int_t nxt_http_js_ext_get_header(njs_vm_t *vm, njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval, njs_value_t *retval); +static njs_int_t nxt_http_js_ext_keys_header(njs_vm_t *vm, + njs_value_t *value, njs_value_t *keys); static njs_int_t nxt_http_js_ext_get_cookie(njs_vm_t *vm, njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval, njs_value_t *retval); +static njs_int_t nxt_http_js_ext_keys_cookie(njs_vm_t *vm, njs_value_t *value, + njs_value_t *keys); static njs_external_t nxt_http_js_proto[] = { @@ -55,12 +59,11 @@ static njs_external_t nxt_http_js_proto[] = { }, { - .flags = NJS_EXTERN_OBJECT, + .flags = NJS_EXTERN_PROPERTY, .name.string = njs_str("args"), .enumerable = 1, - .u.object = { - .enumerable = 1, - .prop_handler = nxt_http_js_ext_get_arg, + .u.property = { + .handler = nxt_http_js_ext_get_args, } }, @@ -71,6 +74,7 @@ static njs_external_t nxt_http_js_proto[] = { .u.object = { .enumerable = 1, .prop_handler = nxt_http_js_ext_get_header, + .keys = nxt_http_js_ext_keys_header, } }, @@ -81,6 +85,7 @@ static njs_external_t nxt_http_js_proto[] = { .u.object = { .enumerable = 1, .prop_handler = nxt_http_js_ext_get_cookie, + .keys = nxt_http_js_ext_keys_cookie, } }, }; @@ -144,14 +149,13 @@ nxt_http_js_ext_remote_addr(njs_vm_t *vm, njs_object_prop_t *prop, static njs_int_t -nxt_http_js_ext_get_arg(njs_vm_t *vm, njs_object_prop_t *prop, +nxt_http_js_ext_get_args(njs_vm_t *vm, njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval, njs_value_t *retval) { - njs_int_t rc; - njs_str_t key; - nxt_array_t *args; - nxt_http_request_t *r; - nxt_http_name_value_t *nv, *start, *end; + njs_int_t ret; + njs_value_t *args; + njs_opaque_value_t val; + nxt_http_request_t *r; r = njs_vm_external(vm, nxt_js_proto_id, value); if (r == NULL) { @@ -159,33 +163,18 @@ nxt_http_js_ext_get_arg(njs_vm_t *vm, njs_object_prop_t *prop, return NJS_DECLINED; } - rc = njs_vm_prop_name(vm, prop, &key); - if (rc != NJS_OK) { - njs_value_undefined_set(retval); - return NJS_DECLINED; - } - - args = nxt_http_arguments_parse(r); - if (nxt_slow_path(args == NULL)) { - return NJS_ERROR; - } - - start = args->elts; - end = start + args->nelts; + args = njs_value_arg(&val); - for (nv = start; nv < end; nv++) { + ret = njs_vm_query_string_parse(vm, r->args->start, + r->args->start + r->args->length, args); - if (key.length == nv->name_length - && memcmp(key.start, nv->name, nv->name_length) == 0) - { - return njs_vm_value_string_set(vm, retval, nv->value, - nv->value_length); - } + if (ret == NJS_ERROR) { + return NJS_ERROR; } - njs_value_undefined_set(retval); + njs_value_assign(retval, args); - return NJS_DECLINED; + return NJS_OK; } @@ -227,6 +216,41 @@ nxt_http_js_ext_get_header(njs_vm_t *vm, njs_object_prop_t *prop, } +static njs_int_t +nxt_http_js_ext_keys_header(njs_vm_t *vm, njs_value_t *value, njs_value_t *keys) +{ + njs_int_t rc; + nxt_http_field_t *f; + nxt_http_request_t *r; + + rc = njs_vm_array_alloc(vm, keys, 4); + if (rc != NJS_OK) { + return NJS_ERROR; + } + + r = njs_vm_external(vm, nxt_js_proto_id, value); + if (r == NULL) { + return NJS_OK; + } + + nxt_list_each(f, r->fields) { + + value = njs_vm_array_push(vm, keys); + if (value == NULL) { + return NJS_ERROR; + } + + rc = njs_vm_value_string_set(vm, value, f->name, f->name_length); + if (rc != NJS_OK) { + return NJS_ERROR; + } + + } nxt_list_loop; + + return NJS_OK; +} + + static njs_int_t nxt_http_js_ext_get_cookie(njs_vm_t *vm, njs_object_prop_t *prop, njs_value_t *value, njs_value_t *setval, njs_value_t *retval) @@ -271,3 +295,46 @@ nxt_http_js_ext_get_cookie(njs_vm_t *vm, njs_object_prop_t *prop, return NJS_DECLINED; } + + +static njs_int_t +nxt_http_js_ext_keys_cookie(njs_vm_t *vm, njs_value_t *value, njs_value_t *keys) +{ + njs_int_t rc; + nxt_array_t *cookies; + nxt_http_request_t *r; + nxt_http_name_value_t *nv, *start, *end; + + rc = njs_vm_array_alloc(vm, keys, 4); + if (rc != NJS_OK) { + return NJS_ERROR; + } + + r = njs_vm_external(vm, nxt_js_proto_id, value); + if (r == NULL) { + return NJS_OK; + } + + cookies = nxt_http_cookies_parse(r); + if (nxt_slow_path(cookies == NULL)) { + return NJS_ERROR; + } + + start = cookies->elts; + end = start + cookies->nelts; + + for (nv = start; nv < end; nv++) { + + value = njs_vm_array_push(vm, keys); + if (value == NULL) { + return NJS_ERROR; + } + + rc = njs_vm_value_string_set(vm, value, nv->name, nv->name_length); + if (rc != NJS_OK) { + return NJS_ERROR; + } + } + + return NJS_OK; +} -- cgit From 0686740f2053abf2b398c6620bb2e74090209fc6 Mon Sep 17 00:00:00 2001 From: Alejandro Colomar Date: Fri, 27 Jan 2023 14:14:43 +0100 Subject: PHP: Factored out code into a helper function. We're going to use zend_stream_init_filename in a following commit. To reduce the diff of that change, move the current code that will be replaced, to a function that has the same interface. We use strlen(3) here to be able to use an interface without passing the length, but we will remove that call in a following code, so it has no performance issues. Co-developed-by: Andrew Clayton Signed-off-by: Alejandro Colomar Reviewed-by: Andrew Clayton Cc: Andrei Zeliankou Signed-off-by: Andrew Clayton --- src/nxt_php_sapi.c | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index d2494938..0358ebdf 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -106,6 +106,8 @@ static nxt_int_t nxt_php_do_301(nxt_unit_request_info_t *req); static void nxt_php_request_handler(nxt_unit_request_info_t *req); static void nxt_php_dynamic_request(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r); +static void nxt_zend_stream_init_filename(zend_file_handle *handle, + const char *filename); static void nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r); nxt_inline void nxt_php_vcwd_chdir(nxt_unit_request_info_t *req, u_char *dir); @@ -1109,6 +1111,21 @@ nxt_php_dynamic_request(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) } +static void +nxt_zend_stream_init_filename(zend_file_handle *handle, const char *filename) +{ + nxt_memzero(handle, sizeof(zend_file_handle)); + + handle->type = ZEND_HANDLE_FILENAME; +#if (PHP_VERSION_ID >= 80100) + handle->filename = zend_string_init(filename, strlen(filename), 0); + handle->primary_script = 1; +#else + handle->filename = filename; +#endif +} + + static void nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) { @@ -1179,16 +1196,8 @@ nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) nxt_php_vcwd_chdir(ctx->req, ctx->script_dirname.start); } - nxt_memzero(&file_handle, sizeof(file_handle)); - - file_handle.type = ZEND_HANDLE_FILENAME; -#if (PHP_VERSION_ID >= 80100) - file_handle.filename = zend_string_init((char *) ctx->script_filename.start, - ctx->script_filename.length, 0); - file_handle.primary_script = 1; -#else - file_handle.filename = (char *) ctx->script_filename.start; -#endif + nxt_zend_stream_init_filename(&file_handle, + (const char *) ctx->script_filename.start); php_execute_script(&file_handle TSRMLS_CC); -- cgit From 3923de987efa79645b3e30a55adca4375e0843e2 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Fri, 27 Jan 2023 14:16:56 +0000 Subject: PHP: Make use of zend_stream_init_filename(). Where possible make use of the zend_stream_init_filename() function introduced in PHP 7.4. This is essentially a preparatory patch for switching to using an already opened file-pointer in nxt_php_execute(). While wrapping this new code in a PHP version check with a fallback to our own function is perhaps slightly overkill, it does reduce the diff of the commit that switches to a FILE *. Reviewed-by: Alejandro Colomar Cc: Andrei Zeliankou Signed-off-by: Andrew Clayton --- src/nxt_php_sapi.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 0358ebdf..0445533e 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -106,8 +106,10 @@ static nxt_int_t nxt_php_do_301(nxt_unit_request_info_t *req); static void nxt_php_request_handler(nxt_unit_request_info_t *req); static void nxt_php_dynamic_request(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r); +#if (PHP_VERSION_ID < 70400) static void nxt_zend_stream_init_filename(zend_file_handle *handle, const char *filename); +#endif static void nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r); nxt_inline void nxt_php_vcwd_chdir(nxt_unit_request_info_t *req, u_char *dir); @@ -1111,19 +1113,17 @@ nxt_php_dynamic_request(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) } +#if (PHP_VERSION_ID < 70400) static void nxt_zend_stream_init_filename(zend_file_handle *handle, const char *filename) { nxt_memzero(handle, sizeof(zend_file_handle)); - handle->type = ZEND_HANDLE_FILENAME; -#if (PHP_VERSION_ID >= 80100) - handle->filename = zend_string_init(filename, strlen(filename), 0); - handle->primary_script = 1; -#else handle->filename = filename; -#endif } +#else +#define nxt_zend_stream_init_filename zend_stream_init_filename +#endif static void -- cgit From fafdb7a57ad976e048147fe23079dca9c602e88a Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Wed, 25 Jan 2023 21:46:01 +0000 Subject: PHP: Simplify ctx->script_filename.start in nxt_php_execute(). Create a const char *filename variable to hold ctx->script_filename.start, which is a much more manageable name and will negate the need for any more casting in the following commit when we switch to using a FILE * instead of a filename in php_execute_script(). Reviewed-by: Alejandro Colomar Cc: Andrei Zeliankou Signed-off-by: Andrew Clayton --- src/nxt_php_sapi.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 0445533e..b517f7c3 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -1132,11 +1132,13 @@ nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) #if (PHP_VERSION_ID < 50600) void *read_post; #endif + const char *filename; nxt_unit_field_t *f; zend_file_handle file_handle; - nxt_unit_req_debug(ctx->req, "PHP execute script %s", - ctx->script_filename.start); + filename = (const char *) ctx->script_filename.start; + + nxt_unit_req_debug(ctx->req, "PHP execute script %s", filename); SG(server_context) = ctx; SG(options) |= SAPI_OPTION_NO_CHDIR; @@ -1196,8 +1198,7 @@ nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) nxt_php_vcwd_chdir(ctx->req, ctx->script_dirname.start); } - nxt_zend_stream_init_filename(&file_handle, - (const char *) ctx->script_filename.start); + nxt_zend_stream_init_filename(&file_handle, filename); php_execute_script(&file_handle TSRMLS_CC); -- cgit From bebc03c729df1d7efc81a5af8b8dac40b333d408 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Thu, 26 Jan 2023 12:09:43 +0000 Subject: PHP: Implement better error handling. Previously the PHP module would produce one of four status codes 200 OK 301 Moved Permanently 500 Internal Server Error 503 Service Unavailable 200 for successful requests, 301 for cases where the url was a directory without a trailing '/', 500 for bad PHP or non-existing PHP file and 503 for all other errors. With this commit we now handle missing files and directories, returning 404 Not Found and files and directories that don't allow access, returning 403 Forbidden. We do these checks in two places, when we check if we should do a directory redirect (bar -> bar/) and in the nxt_php_execute() function. One snag with the latter is that the php_execute_script() function only returns success/failure (no reason). However while it took a zend_file_handle structure with the filename of the script to run, we can instead pass through an already opened file-pointer (FILE *) via that structure. So we can try opening the script ourselves and do the required checks before calling php_execute_script(). We also make use of the zend_stream_init_fp() function that initialises the zend_file_handle structure if it's available otherwise we use our own version. This is good because the zend_file_handle structure has changed over time and the zend_stream_init_fp() function should change with it. Closes: Reviewed-by: Alejandro Colomar Cc: Andrei Zeliankou Signed-off-by: Andrew Clayton --- src/nxt_php_sapi.c | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index b517f7c3..32a13a70 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -102,12 +102,13 @@ static void nxt_php_str_trim_lead(nxt_str_t *str, u_char t); nxt_inline u_char *nxt_realpath(const void *c); static nxt_int_t nxt_php_do_301(nxt_unit_request_info_t *req); +static nxt_int_t nxt_php_handle_fs_err(nxt_unit_request_info_t *req); static void nxt_php_request_handler(nxt_unit_request_info_t *req); static void nxt_php_dynamic_request(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r); #if (PHP_VERSION_ID < 70400) -static void nxt_zend_stream_init_filename(zend_file_handle *handle, +static void nxt_zend_stream_init_fp(zend_file_handle *handle, FILE *fp, const char *filename); #endif static void nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r); @@ -984,6 +985,24 @@ nxt_php_do_301(nxt_unit_request_info_t *req) } +static nxt_int_t +nxt_php_handle_fs_err(nxt_unit_request_info_t *req) +{ + switch (nxt_errno) { + case ELOOP: + case EACCES: + case ENFILE: + return nxt_unit_response_init(req, NXT_HTTP_FORBIDDEN, 0, 0); + case ENOENT: + case ENOTDIR: + case ENAMETOOLONG: + return nxt_unit_response_init(req, NXT_HTTP_NOT_FOUND, 0, 0); + } + + return NXT_UNIT_ERROR; +} + + static void nxt_php_request_handler(nxt_unit_request_info_t *req) { @@ -1062,6 +1081,8 @@ nxt_php_dynamic_request(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) ret = stat(tpath, &sb); if (ret == 0 && S_ISDIR(sb.st_mode)) { ec = nxt_php_do_301(ctx->req); + } else if (ret == -1) { + ec = nxt_php_handle_fs_err(ctx->req); } nxt_unit_request_done(ctx->req, ec); @@ -1115,20 +1136,23 @@ nxt_php_dynamic_request(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) #if (PHP_VERSION_ID < 70400) static void -nxt_zend_stream_init_filename(zend_file_handle *handle, const char *filename) +nxt_zend_stream_init_fp(zend_file_handle *handle, FILE *fp, + const char *filename) { nxt_memzero(handle, sizeof(zend_file_handle)); - handle->type = ZEND_HANDLE_FILENAME; + handle->type = ZEND_HANDLE_FP; + handle->handle.fp = fp; handle->filename = filename; } #else -#define nxt_zend_stream_init_filename zend_stream_init_filename +#define nxt_zend_stream_init_fp zend_stream_init_fp #endif static void nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) { + FILE *fp; #if (PHP_VERSION_ID < 50600) void *read_post; #endif @@ -1140,6 +1164,17 @@ nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) nxt_unit_req_debug(ctx->req, "PHP execute script %s", filename); + fp = fopen(filename, "re"); + if (fp == NULL) { + nxt_int_t ec; + + nxt_unit_req_debug(ctx->req, "PHP fopen(\"%s\") failed", filename); + + ec = nxt_php_handle_fs_err(ctx->req); + nxt_unit_request_done(ctx->req, ec); + return; + } + SG(server_context) = ctx; SG(options) |= SAPI_OPTION_NO_CHDIR; SG(request_info).request_uri = nxt_unit_sptr_get(&r->target); @@ -1198,7 +1233,7 @@ nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) nxt_php_vcwd_chdir(ctx->req, ctx->script_dirname.start); } - nxt_zend_stream_init_filename(&file_handle, filename); + nxt_zend_stream_init_fp(&file_handle, fp, filename); php_execute_script(&file_handle TSRMLS_CC); -- cgit From 5a37171f733fa8c7326161cc49af3df0be5dfae4 Mon Sep 17 00:00:00 2001 From: Alejandro Colomar Date: Thu, 14 Jul 2022 13:25:57 +0200 Subject: Added default values for pathnames. This allows one to simply run `./configure` and expect it to produce sane defaults for an install. Previously, without specifying `--prefix=...`, `make install` would simply fail, recommending to set `--prefix` or `DESTDIR`, but that recommendation was incomplete at best, since it didn't set many of the subdirs needed for a good organization. Setting `DESTDIR` was even worse, since that shouldn't even affect an installation (it is required to be transparent to the installation). /usr/local is the historic Unix standard path to use for installations from source made manually by the admin of the system. Some package managers (Homebrew, I'm looking specifically at you) have abused that path to install their things, but 1) it's not our fault that someone else incorrectly abuses that path (and they seem to be fixing it for newer archs; e.g., they started using /opt/homebrew for Apple Silicon), 2) there's no better path than /usr/local, 3) we still allow changing it for systems where this might not be the desired path (MacOS Intel with hombrew), and 4) it's _the standard_. See a related conversation with Ingo (OpenBSD maintainer): On 7/27/22 16:16, Ingo Schwarze wrote: > Hi Alejandro, [...] > > Alejandro Colomar wrote on Sun, Jul 24, 2022 at 07:07:18PM +0200: >> On 7/24/22 16:57, Ingo Schwarze wrote: >>> Alejandro Colomar wrote on Sun, Jul 24, 2022 at 01:20:46PM +0200: > >>>> /usr/local is for sysadmins to build from source; > >>> Doing that is *very* strongly discouraged on OpenBSD. > >> I guess that's why the directory was reused in the BSDs to install ports >> (probably ports were installed by the sysadmin there, and by extension, >> ports are now always installed there, but that's just a guess). > > Maybe. In any case, the practice of using /usr/local for packages > created from ports is significantly older than the recommendation > to refrain from using upstream "make install" outside the ports > framework. > > * The FreeBSD ports framework was started by Jordan Hubbard in 1993. > * The ports framework was ported from FreeBSD to OpenBSD > by Niklas Hallqvist in 1996. > * NetBSD pkgsrc was forked from FreeBSD ports by Alistair G. Crooks > and Hubert Feyrer in 1997. > > I failed to quickly find Jordan's original version, but rev. 1.1 > of /usr/ports/infrastructure/mk/bsd.port.mk in OpenBSD (dated Jun 3 > 22:47:10 1996 UTC) already said > > LOCALBASE ?= /usr/local > PREFIX ?= ${LOCALBASE} > [...] >> I had a discussion in NGINX Unit about it, and >> the decission for now has been: "support prefix=/usr/local for default >> manual installation through the Makefile, and let BSD users adjust to >> their preferred path". > > That's an *excellent* solution for the task, thanks for doing it > the right way. By setting PREFIX=/usr/local by default in the > upstream Makefile, you are minimizing the work for *BSD porters. > > The BSD ports frameworks will typically run the upstreak "make install" > with the variable DESTDIR set to a custom value, for example > > DESTDIR=/usr/ports/pobj/groff-1.23.0/fake-amd64 > > so if the upstream Makefile sets PREFIX=/usr/local , > that's perfect, everything gets installed to the right place > without an intervention by the person doing the porting. > > Of course, if the upstream Makefile would use some other PREFIX, > that would not be a huge obstacle. All we have to do in that case > is pass the option --prefix=/usr/local to the ./configure script, > or something equivalent if the software isn't using GNU configure. > >> We were concerned that we might get collisions >> with the BSD port also installing in /usr/local, but that's the least >> evil (and considering BSD users don't typically run `make install`, it's >> not so bad). > > It's not bad at all. It's perfect. > > Of course, if a user wants to install *without* the ports framework, > they have to provide their own --prefix. But that's not an issue > because it is easy to do, and installing without a port is discouraged > anyway. === Directory variables should never contain a trailing slash (I've learned that the hard way, where some things would break unexpectedly). Especially, make(1) is likely to have problems when things have double slashes or a trailing slash, since it treats filenames as text strings. I've removed the trailing slash from the prefix, and added it to the derivate variables just after the prefix. pkg-config(1) also expects directory variables to have no trailing slash. === I also removed the code that would set variables as depending on the prefix if they didn't start with a slash, because that is a rather non-obvious behavior, and things should not always depend on prefix, but other dirs such as $(runstatedir), so if we keep a similar behavior it would be very unreliable. Better keep variables intact if set, or use the default if unset. === Print the real defaults for ./configure --help, rather than the actual values. === I used a subdirectory under the standard /var/lib for NXT_STATE, instead of a homemade "state" dir that does the same thing. === Modified the Makefile to create some dirs that weren't being created, and also remove those that weren't being removed in uninstall, probably because someone forgot to add them. === Add new options for setting the new variables, and rename some to be consistent with the standard names. Keep the old ones at configuration time for compatibility, but mark them as deprecated. Don't keep the old ones at exec time. === A summary of the default config is: Unit configuration summary: bin directory: ............. "/usr/local/bin" sbin directory: ............ "/usr/local/sbin" lib directory: ............. "/usr/local/lib" include directory: ......... "/usr/local/include" man pages directory: ....... "/usr/local/share/man" modules directory: ......... "/usr/local/lib/unit/modules" state directory: ........... "/usr/local/var/lib/unit" tmp directory: ............. "/tmp" pid file: .................. "/usr/local/var/run/unit/unit.pid" log file: .................. "/usr/local/var/log/unit/unit.log" control API socket: ........ "unix:/usr/local/var/run/unit/control.unit.sock" Link: Link: Reviewed-by: Artem Konev Reviewed-by: Andrew Clayton Tested-by: Andrew Clayton Reviewed-by: Konstantin Pavlov Signed-off-by: Alejandro Colomar --- src/nxt_runtime.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index c7e4455e..cb5fe421 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -768,10 +768,10 @@ nxt_runtime_conf_init(nxt_task_t *task, nxt_runtime_t *rt) rt->group = NXT_GROUP; rt->pid = NXT_PID; rt->log = NXT_LOG; - rt->modules = NXT_MODULES; - rt->state = NXT_STATE; + rt->modules = NXT_MODULESDIR; + rt->state = NXT_LIBSTATEDIR; rt->control = NXT_CONTROL_SOCK; - rt->tmp = NXT_TMP; + rt->tmp = NXT_TMPDIR; nxt_memzero(&rt->capabilities, sizeof(nxt_capabilities_t)); @@ -927,9 +927,10 @@ nxt_runtime_conf_read_cmd(nxt_task_t *task, nxt_runtime_t *rt) static const char no_pid[] = "option \"--pid\" requires filename\n"; static const char no_log[] = "option \"--log\" requires filename\n"; static const char no_modules[] = - "option \"--modules\" requires directory\n"; - static const char no_state[] = "option \"--state\" requires directory\n"; - static const char no_tmp[] = "option \"--tmp\" requires directory\n"; + "option \"--modulesdir\" requires directory\n"; + static const char no_state[] = + "option \"--libstatedir\" requires directory\n"; + static const char no_tmp[] = "option \"--tmpdir\" requires directory\n"; static const char help[] = "\n" @@ -948,14 +949,14 @@ nxt_runtime_conf_read_cmd(nxt_task_t *task, nxt_runtime_t *rt) " --log FILE set log filename\n" " default: \"" NXT_LOG "\"\n" "\n" - " --modules DIRECTORY set modules directory name\n" - " default: \"" NXT_MODULES "\"\n" + " --modulesdir DIR set modules directory name\n" + " default: \"" NXT_MODULESDIR "\"\n" "\n" - " --state DIRECTORY set state directory name\n" - " default: \"" NXT_STATE "\"\n" + " --libstatedir DIR set state directory name\n" + " default: \"" NXT_LIBSTATEDIR "\"\n" "\n" - " --tmp DIRECTORY set tmp directory name\n" - " default: \"" NXT_TMP "\"\n" + " --tmpdir DIR set tmp directory name\n" + " default: \"" NXT_TMPDIR "\"\n" "\n" " --user USER set non-privileged processes to run" " as specified user\n" @@ -1038,7 +1039,7 @@ nxt_runtime_conf_read_cmd(nxt_task_t *task, nxt_runtime_t *rt) continue; } - if (nxt_strcmp(p, "--modules") == 0) { + if (nxt_strcmp(p, "--modulesdir") == 0) { if (*argv == NULL) { write(STDERR_FILENO, no_modules, nxt_length(no_modules)); return NXT_ERROR; @@ -1051,7 +1052,7 @@ nxt_runtime_conf_read_cmd(nxt_task_t *task, nxt_runtime_t *rt) continue; } - if (nxt_strcmp(p, "--state") == 0) { + if (nxt_strcmp(p, "--libstatedir") == 0) { if (*argv == NULL) { write(STDERR_FILENO, no_state, nxt_length(no_state)); return NXT_ERROR; @@ -1064,7 +1065,7 @@ nxt_runtime_conf_read_cmd(nxt_task_t *task, nxt_runtime_t *rt) continue; } - if (nxt_strcmp(p, "--tmp") == 0) { + if (nxt_strcmp(p, "--tmpdir") == 0) { if (*argv == NULL) { write(STDERR_FILENO, no_tmp, nxt_length(no_tmp)); return NXT_ERROR; -- cgit From 4b7a95469cf309b437fc1a8201a93f14c33ce3a8 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Fri, 20 Jan 2023 03:27:59 +0000 Subject: Python: ASGI: Factor out event loop creation to its own function. This is a preparatory patch that factors out the asyncio event loop creation code from nxt_python_asgi_ctx_data_alloc() into its own function, to facilitate being called multiple times. This a part of the work to move away from using the asyncio.get_event_loop() function due to it no longer creating event loops if there wasn't one running. See the following commit for the gory details. Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/python/nxt_python_asgi.c | 56 +++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 21 deletions(-) (limited to 'src') diff --git a/src/python/nxt_python_asgi.c b/src/python/nxt_python_asgi.c index 587a17cf..716814a8 100644 --- a/src/python/nxt_python_asgi.c +++ b/src/python/nxt_python_asgi.c @@ -17,6 +17,8 @@ static PyObject *nxt_python_asgi_get_func(PyObject *obj); +static PyObject *nxt_python_asgi_get_event_loop(PyObject *asyncio, + const char *event_loop_func); static int nxt_python_asgi_ctx_data_alloc(void **pdata, int main); static void nxt_python_asgi_ctx_data_free(void *data); static int nxt_python_asgi_startup(void *data); @@ -201,11 +203,41 @@ nxt_python_asgi_init(nxt_unit_init_t *init, nxt_python_proto_t *proto) } +static PyObject * +nxt_python_asgi_get_event_loop(PyObject *asyncio, const char *event_loop_func) +{ + PyObject *event_loop, *loop; + + event_loop = PyDict_GetItemString(PyModule_GetDict(asyncio), + event_loop_func); + if (nxt_slow_path(event_loop == NULL)) { + nxt_unit_alert(NULL, "Python failed to get '%s' from module 'asyncio'", + event_loop_func); + return NULL; + } + + if (nxt_slow_path(PyCallable_Check(event_loop) == 0)) { + nxt_unit_alert(NULL, "'asyncio.%s' is not a callable object", + event_loop_func); + return NULL; + } + + loop = PyObject_CallObject(event_loop, NULL); + if (nxt_slow_path(loop == NULL)) { + nxt_unit_alert(NULL, "Python failed to call 'asyncio.%s'", + event_loop_func); + return NULL; + } + + return loop; +} + + static int nxt_python_asgi_ctx_data_alloc(void **pdata, int main) { uint32_t i; - PyObject *asyncio, *loop, *event_loop, *obj; + PyObject *asyncio, *loop, *obj; const char *event_loop_func; nxt_py_asgi_ctx_data_t *ctx_data; @@ -243,26 +275,8 @@ nxt_python_asgi_ctx_data_alloc(void **pdata, int main) event_loop_func = main ? "get_event_loop" : "new_event_loop"; - event_loop = PyDict_GetItemString(PyModule_GetDict(asyncio), - event_loop_func); - if (nxt_slow_path(event_loop == NULL)) { - nxt_unit_alert(NULL, - "Python failed to get '%s' from module 'asyncio'", - event_loop_func); - goto fail; - } - - if (nxt_slow_path(PyCallable_Check(event_loop) == 0)) { - nxt_unit_alert(NULL, - "'asyncio.%s' is not a callable object", - event_loop_func); - goto fail; - } - - loop = PyObject_CallObject(event_loop, NULL); - if (nxt_slow_path(loop == NULL)) { - nxt_unit_alert(NULL, "Python failed to call 'asyncio.%s'", - event_loop_func); + loop = nxt_python_asgi_get_event_loop(asyncio, event_loop_func); + if (loop == NULL) { goto fail; } -- cgit From 9c0a4a09978e0defbc6656c7866a7ca147236733 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Fri, 20 Jan 2023 03:33:37 +0000 Subject: Python: ASGI: Switch away from asyncio.get_event_loop(). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several users on GitHub reported issues with running Python ASGI apps on Unit with Python 3.11.1 (this would also effect Python 3.10.9) with the following error from Unit 2023/01/15 22:43:22 [alert] 0#77128 [unit] Python failed to call 'asyncio.get_event_loop' TL;DR asyncio.get_event_loop() is currently broken due to the process of deprecating part or all of it. First some history. In Unit we had this commit commit 8dcb0b9987033d0349a6ecf528014a9daa574787 Author: Max Romanov Date: Thu Nov 5 00:04:59 2020 +0300 Python: request processing in multiple threads. One of things this did was to create a new asyncio event loop in each thread using asyncio.new_event_loop(). It's perhaps worth noting that all these asyncio.* functions are Python functions that we call from the C code in Unit. Then we had this commit commit f27fbd9b4d2bdaddf1e7001d0d0bc5586ba04cd4 Author: Max Romanov Date: Tue Jul 20 10:37:54 2021 +0300 Python: using default event_loop for main thread for ASGI. This changed things so that Unit calls asyncio.get_event_loop() in the _main_ thread (but still calls asyncio.new_event_loop() in the other threads). asyncio.get_event_loop() up until recently would either return an already running event loop or return a newly created one. This was done for $reasons that the commit message and GitHub issue #560 hint at. But the intimation is that there can already be an event loop running from the application (I assume it's referring to the users application) at this point and if there is we should use it. Now for the Python side of things. On the main branch we had commit 172c0f2752d8708b6dda7b42e6c5a3519420a4e8 Author: Serhiy Storchaka Date: Sun Apr 25 13:40:44 2021 +0300 bpo-39529: Deprecate creating new event loop in asyncio.get_event_loop() (GH-23554) This commit began the deprecating of asyncio.get_event_loop(). commit fd38a2f0ec03b4eec5e3cfd41241d198b1ee555a Author: Serhiy Storchaka Date: Tue Dec 6 19:42:12 2022 +0200 gh-93453: No longer create an event loop in get_event_loop() (#98440) This turned asyncio.get_event_loop() into a RuntimeError _if_ there isn't a current event loop. commit e5bd5ad70d9e549eeb80aadb4f3ccb0f2f23266d Author: Serhiy Storchaka Date: Fri Jan 13 14:40:29 2023 +0200 gh-100160: Restore and deprecate implicit creation of an event loop (GH-100410) This re-creates the event loop if there wasn't one and emits a deprecation warning. After at least the last two commits Unit no longer works with the Python _main_ branch. Meanwhile on the 3.11 branch we had commit 3fae04b10e2655a20a3aadb5e0d63e87206d0c67 Author: Serhiy Storchaka Date: Tue Dec 6 17:15:44 2022 +0200 [3.11] gh-93453: Only emit deprecation warning in asyncio.get_event_loop when a new event loop is created (#99949) which is what caused our breakage, though perhaps unintentionally as we get the following traceback Traceback (most recent call last): File "/usr/lib64/python3.11/asyncio/events.py", line 676, in get_event_loop f = sys._getframe(1) ^^^^^^^^^^^^^^^^ ValueError: call stack is not deep enough 2023/01/18 02:46:10 [alert] 0#180279 [unit] Python failed to call 'asyncio.get_event_loop' However, regardless, it is clear we need to stop using asyncio.get_event_loop(). One option is to switch to the higher level asyncio.run() API, however that is a rather large change. This commit takes the simpler approach of using asyncio.get_running_loop() (which it seems get_event_loop() will eventually be an alias of) in the _main_ thread to return the currently running event loop, or if there is no current event loop, it will call asyncio.new_event_loop() to return a newly created event loop. I believe this mimics the current behaviour. In my testing get_event_loop() seemed to always return a newly created loop, as when just calling get_running_loop() it would return NULL and we would fail out. When running two processes each with 2 threads we would get the following loops with Python 3.11.0 and unpatched Unit <_UnixSelectorEventLoop running=False closed=False debug=False> <_UnixSelectorEventLoop running=False closed=False debug=False> <_UnixSelectorEventLoop running=False closed=False debug=False> <_UnixSelectorEventLoop running=False closed=False debug=False> and with Python 3.11.1 and a patched Unit we would get <_UnixSelectorEventLoop running=False closed=False debug=False> <_UnixSelectorEventLoop running=False closed=False debug=False> <_UnixSelectorEventLoop running=False closed=False debug=False> <_UnixSelectorEventLoop running=False closed=False debug=False> Tested-by: RafaƂ Safin Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/python/nxt_python_asgi.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/python/nxt_python_asgi.c b/src/python/nxt_python_asgi.c index 716814a8..f26f5a5d 100644 --- a/src/python/nxt_python_asgi.c +++ b/src/python/nxt_python_asgi.c @@ -241,6 +241,12 @@ nxt_python_asgi_ctx_data_alloc(void **pdata, int main) const char *event_loop_func; nxt_py_asgi_ctx_data_t *ctx_data; +#if PY_VERSION_HEX < NXT_PYTHON_VER(3, 7) + static const char *main_event_loop_func = "get_event_loop"; +#else + static const char *main_event_loop_func = "get_running_loop"; +#endif + ctx_data = nxt_unit_malloc(NULL, sizeof(nxt_py_asgi_ctx_data_t)); if (nxt_slow_path(ctx_data == NULL)) { nxt_unit_alert(NULL, "Failed to allocate context data"); @@ -273,11 +279,24 @@ nxt_python_asgi_ctx_data_alloc(void **pdata, int main) goto fail; } - event_loop_func = main ? "get_event_loop" : "new_event_loop"; + event_loop_func = main ? main_event_loop_func : "new_event_loop"; loop = nxt_python_asgi_get_event_loop(asyncio, event_loop_func); if (loop == NULL) { +#if PY_VERSION_HEX < NXT_PYTHON_VER(3, 7) goto fail; +#else + if (!main) { + goto fail; + } + + PyErr_Clear(); + + loop = nxt_python_asgi_get_event_loop(asyncio, "new_event_loop"); + if (nxt_slow_path(loop == NULL)) { + goto fail; + } +#endif } for (i = 0; i < nxt_nitems(handlers); i++) { -- cgit From cbc01907fef6a028f02d9051b909def920c49f24 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Tue, 7 Feb 2023 13:11:10 +0000 Subject: Python: ASGI: Don't log asyncio.get_running_loop() errors. This adds a check to nxt_python_asgi_get_event_loop() on the event_loop_func name in the case that running that function fails, and if it's get_running_loop() that failed we skip printing an error message as this is an often expected behaviour since the previous commit and we don't want users reporting erroneous bugs. This check will always happen regardless of Python version while it really only applies to Python >= 3.7, there didn't seem much point adding complexity to the code for this case and in what will be an ever diminishing case of people running older Pythons. Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/python/nxt_python_asgi.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/python/nxt_python_asgi.c b/src/python/nxt_python_asgi.c index f26f5a5d..adf03e2b 100644 --- a/src/python/nxt_python_asgi.c +++ b/src/python/nxt_python_asgi.c @@ -224,8 +224,11 @@ nxt_python_asgi_get_event_loop(PyObject *asyncio, const char *event_loop_func) loop = PyObject_CallObject(event_loop, NULL); if (nxt_slow_path(loop == NULL)) { - nxt_unit_alert(NULL, "Python failed to call 'asyncio.%s'", - event_loop_func); + if (strcmp(event_loop_func, "get_running_loop") != 0) { + nxt_unit_alert(NULL, "Python failed to call 'asyncio.%s'", + event_loop_func); + } + return NULL; } -- cgit From 789762ff3d88a1c006babc7a8e7037e0976ad70f Mon Sep 17 00:00:00 2001 From: Zhidao HONG Date: Mon, 30 Jan 2023 11:16:01 +0800 Subject: NJS: adding the missing vm destruction. This commit fixed the njs memory leak happened in the config validation, updating and http requests. --- src/nxt_conf_validation.c | 18 ++++++++++++++---- src/nxt_http_request.c | 4 ++++ src/nxt_js.c | 17 +++++++++++++++++ src/nxt_js.h | 2 ++ src/nxt_router.c | 6 ++++++ src/nxt_tstr.c | 18 ++++++++++++++++++ src/nxt_tstr.h | 2 ++ 7 files changed, 63 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index bf8aa760..537a3fb7 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -1284,25 +1284,35 @@ nxt_conf_validate(nxt_conf_validation_t *vldt) vldt->tstr_state = nxt_tstr_state_new(vldt->pool, 1); if (nxt_slow_path(vldt->tstr_state == NULL)) { - return NXT_ERROR; + ret = NXT_ERROR; + goto fail; } ret = nxt_conf_vldt_type(vldt, NULL, vldt->conf, NXT_CONF_VLDT_OBJECT); if (ret != NXT_OK) { - return ret; + goto fail; } ret = nxt_conf_vldt_object(vldt, vldt->conf, nxt_conf_vldt_root_members); if (ret != NXT_OK) { - return ret; + goto fail; } ret = nxt_tstr_state_done(vldt->tstr_state, error); if (ret != NXT_OK) { - return nxt_conf_vldt_error(vldt, "%s", error); + ret = nxt_conf_vldt_error(vldt, "%s", error); + goto fail; } + nxt_tstr_state_release(vldt->tstr_state); + return NXT_OK; + +fail: + + nxt_tstr_state_release(vldt->tstr_state); + + return ret; } diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index 73ffd2f0..e78975aa 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -833,6 +833,10 @@ nxt_http_request_close_handler(nxt_task_t *task, void *obj, void *data) r->body->file->fd = -1; } + if (r->tstr_query != NULL) { + nxt_tstr_query_release(r->tstr_query); + } + if (nxt_fast_path(proto.any != NULL)) { protocol = r->protocol; diff --git a/src/nxt_js.c b/src/nxt_js.c index aa3c4af5..4327e848 100644 --- a/src/nxt_js.c +++ b/src/nxt_js.c @@ -46,6 +46,7 @@ nxt_js_conf_new(nxt_mp_t *mp) jcf->funcs = nxt_array_create(mp, 4, sizeof(nxt_str_t)); if (nxt_slow_path(jcf->funcs == NULL)) { + njs_vm_destroy(jcf->vm); return NULL; } @@ -53,6 +54,13 @@ nxt_js_conf_new(nxt_mp_t *mp) } +void +nxt_js_conf_release(nxt_js_conf_t *jcf) +{ + njs_vm_destroy(jcf->vm); +} + + void nxt_js_set_proto(nxt_js_conf_t *jcf, njs_external_t *proto, njs_uint_t n) { @@ -297,3 +305,12 @@ nxt_js_call(nxt_task_t *task, nxt_js_cache_t *cache, nxt_js_t *js, return NXT_OK; } + + +void +nxt_js_release(nxt_js_cache_t *cache) +{ + if (cache->vm != NULL) { + njs_vm_destroy(cache->vm); + } +} diff --git a/src/nxt_js.h b/src/nxt_js.h index dea43fe3..74d041ca 100644 --- a/src/nxt_js.h +++ b/src/nxt_js.h @@ -22,12 +22,14 @@ typedef struct { nxt_js_conf_t *nxt_js_conf_new(nxt_mp_t *mp); +void nxt_js_conf_release(nxt_js_conf_t *jcf); void nxt_js_set_proto(nxt_js_conf_t *jcf, njs_external_t *proto, nxt_uint_t n); nxt_js_t *nxt_js_add_tpl(nxt_js_conf_t *jcf, nxt_str_t *str, nxt_bool_t strz); nxt_int_t nxt_js_compile(nxt_js_conf_t *jcf); nxt_int_t nxt_js_test(nxt_js_conf_t *jcf, nxt_str_t *str, u_char *error); nxt_int_t nxt_js_call(nxt_task_t *task, nxt_js_cache_t *cache, nxt_js_t *js, nxt_str_t *str, void *ctx); +void nxt_js_release(nxt_js_cache_t *cache); extern njs_int_t nxt_js_proto_id; diff --git a/src/nxt_router.c b/src/nxt_router.c index edc015c5..17f6c572 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1111,6 +1111,10 @@ temp_fail: fail: + if (rtcf->tstr_state != NULL) { + nxt_tstr_state_release(rtcf->tstr_state); + } + nxt_mp_destroy(mp); return NULL; @@ -3794,6 +3798,8 @@ nxt_router_conf_release(nxt_task_t *task, nxt_socket_conf_joint_t *joint) nxt_router_access_log_release(task, lock, rtcf->access_log); + nxt_tstr_state_release(rtcf->tstr_state); + nxt_mp_thread_adopt(rtcf->mem_pool); nxt_mp_destroy(rtcf->mem_pool); diff --git a/src/nxt_tstr.c b/src/nxt_tstr.c index fd01797c..fda585b8 100644 --- a/src/nxt_tstr.c +++ b/src/nxt_tstr.c @@ -194,6 +194,15 @@ nxt_tstr_state_done(nxt_tstr_state_t *state, u_char *error) } +void +nxt_tstr_state_release(nxt_tstr_state_t *state) +{ +#if (NXT_HAVE_NJS) + nxt_js_conf_release(state->jcf); +#endif +} + + nxt_bool_t nxt_tstr_is_const(nxt_tstr_t *tstr) { @@ -315,3 +324,12 @@ nxt_tstr_query_handle(nxt_task_t *task, nxt_tstr_query_t *query, task, query->ctx, query->data); } } + + +void +nxt_tstr_query_release(nxt_tstr_query_t *query) +{ +#if (NXT_HAVE_NJS) + nxt_js_release(&query->cache->js); +#endif +} diff --git a/src/nxt_tstr.h b/src/nxt_tstr.h index 0cc24292..ce8e6f3a 100644 --- a/src/nxt_tstr.h +++ b/src/nxt_tstr.h @@ -42,6 +42,7 @@ nxt_tstr_t *nxt_tstr_compile(nxt_tstr_state_t *state, nxt_str_t *str, nxt_tstr_flags_t flags); nxt_int_t nxt_tstr_test(nxt_tstr_state_t *state, nxt_str_t *str, u_char *error); nxt_int_t nxt_tstr_state_done(nxt_tstr_state_t *state, u_char *error); +void nxt_tstr_state_release(nxt_tstr_state_t *state); nxt_bool_t nxt_tstr_is_const(nxt_tstr_t *tstr); void nxt_tstr_str(nxt_tstr_t *tstr, nxt_str_t *str); @@ -55,6 +56,7 @@ void nxt_tstr_query_resolve(nxt_task_t *task, nxt_tstr_query_t *query, void *data, nxt_work_handler_t ready, nxt_work_handler_t error); void nxt_tstr_query_handle(nxt_task_t *task, nxt_tstr_query_t *query, nxt_bool_t failed); +void nxt_tstr_query_release(nxt_tstr_query_t *query); nxt_inline nxt_bool_t -- cgit From 3ecdd2c69c4864526c63b8e55df22ad1a86f3c72 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Fri, 18 Nov 2022 23:42:44 +0000 Subject: Isolation: Rename NXT_HAVE_CLONE -> NXT_HAVE_LINUX_NS. Due to the need to replace our use of clone/__NR_clone on Linux with fork(2)/unshare(2) for enabling Linux namespaces(7) to keep the pthreads(7) API working. Let's rename NXT_HAVE_CLONE to NXT_HAVE_LINUX_NS, i.e name it after the feature, not how it's implemented, then in future if we change how we do namespaces again we don't have to rename this. Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_clone.c | 2 +- src/nxt_credential.c | 6 +++--- src/nxt_isolation.c | 8 ++++---- src/nxt_main_process.c | 2 +- src/nxt_process.c | 10 +++++----- src/nxt_process.h | 6 +++--- 6 files changed, 17 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/nxt_clone.c b/src/nxt_clone.c index a9b39ac1..a98aac47 100644 --- a/src/nxt_clone.c +++ b/src/nxt_clone.c @@ -8,7 +8,7 @@ #include #include -#if (NXT_HAVE_CLONE) +#if (NXT_HAVE_LINUX_NS) pid_t nxt_clone(nxt_int_t flags) diff --git a/src/nxt_credential.c b/src/nxt_credential.c index 168db9cf..bda97024 100644 --- a/src/nxt_credential.c +++ b/src/nxt_credential.c @@ -286,7 +286,7 @@ nxt_credential_setuid(nxt_task_t *task, nxt_credential_t *uc) if (setuid(uc->uid) != 0) { -#if (NXT_HAVE_CLONE) +#if (NXT_HAVE_LINUX_NS) if (nxt_errno == EINVAL) { nxt_log(task, NXT_LOG_ERR, "The uid %d (user \"%s\") isn't " "valid in the application namespace.", uc->uid, uc->user); @@ -314,7 +314,7 @@ nxt_credential_setgids(nxt_task_t *task, nxt_credential_t *uc) if (setgid(uc->base_gid) != 0) { -#if (NXT_HAVE_CLONE) +#if (NXT_HAVE_LINUX_NS) if (nxt_errno == EINVAL) { nxt_log(task, NXT_LOG_ERR, "The gid %d isn't valid in the " "application namespace.", uc->base_gid); @@ -333,7 +333,7 @@ nxt_credential_setgids(nxt_task_t *task, nxt_credential_t *uc) if (nxt_slow_path(uc->ngroups > 0 && setgroups(uc->ngroups, uc->gids) != 0)) { -#if (NXT_HAVE_CLONE) +#if (NXT_HAVE_LINUX_NS) if (nxt_errno == EINVAL) { nxt_log(task, NXT_LOG_ERR, "The user \"%s\" (uid: %d) has " "supplementary group ids not valid in the application " diff --git a/src/nxt_isolation.c b/src/nxt_isolation.c index b6b13c59..e43cf1f7 100644 --- a/src/nxt_isolation.c +++ b/src/nxt_isolation.c @@ -21,7 +21,7 @@ static nxt_int_t nxt_isolation_set_cgroup(nxt_task_t *task, nxt_conf_value_t *isolation, nxt_process_t *process); #endif -#if (NXT_HAVE_CLONE) +#if (NXT_HAVE_LINUX_NS) static nxt_int_t nxt_isolation_set_namespaces(nxt_task_t *task, nxt_conf_value_t *isolation, nxt_process_t *process); static nxt_int_t nxt_isolation_clone_flags(nxt_task_t *task, @@ -169,7 +169,7 @@ nxt_isolation_set(nxt_task_t *task, nxt_conf_value_t *isolation, } #endif -#if (NXT_HAVE_CLONE) +#if (NXT_HAVE_LINUX_NS) if (nxt_slow_path(nxt_isolation_set_namespaces(task, isolation, process) != NXT_OK)) { @@ -247,7 +247,7 @@ nxt_isolation_set_cgroup(nxt_task_t *task, nxt_conf_value_t *isolation, #endif -#if (NXT_HAVE_CLONE) +#if (NXT_HAVE_LINUX_NS) static nxt_int_t nxt_isolation_set_namespaces(nxt_task_t *task, nxt_conf_value_t *isolation, @@ -409,7 +409,7 @@ nxt_isolation_vldt_creds(nxt_task_t *task, nxt_process_t *process) #endif -#if (NXT_HAVE_CLONE) +#if (NXT_HAVE_LINUX_NS) static nxt_int_t nxt_isolation_clone_flags(nxt_task_t *task, nxt_conf_value_t *namespaces, diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index de41e8d7..4c89121e 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -556,7 +556,7 @@ nxt_main_process_created_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_assert(process != NULL); nxt_assert(process->state == NXT_PROCESS_STATE_CREATING); -#if (NXT_HAVE_CLONE && NXT_HAVE_CLONE_NEWUSER) +#if (NXT_HAVE_LINUX_NS && NXT_HAVE_CLONE_NEWUSER) if (nxt_is_clone_flag_set(process->isolation.clone.flags, NEWUSER)) { if (nxt_slow_path(nxt_clone_credential_map(task, process->pid, process->user_cred, diff --git a/src/nxt_process.c b/src/nxt_process.c index d8836ad2..b40eb8cf 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -7,7 +7,7 @@ #include #include -#if (NXT_HAVE_CLONE) +#if (NXT_HAVE_LINUX_NS) #include #endif @@ -18,7 +18,7 @@ #endif -#if (NXT_HAVE_CLONE) && (NXT_HAVE_CLONE_NEWPID) +#if (NXT_HAVE_LINUX_NS) && (NXT_HAVE_CLONE_NEWPID) #define nxt_is_pid_isolated(process) \ nxt_is_clone_flag_set(process->isolation.clone.flags, NEWPID) #else @@ -318,7 +318,7 @@ nxt_process_create(nxt_task_t *task, nxt_process_t *process) nxt_pid_t pid; nxt_runtime_t *rt; -#if (NXT_HAVE_CLONE) +#if (NXT_HAVE_LINUX_NS) pid = nxt_clone(SIGCHLD | process->isolation.clone.flags); if (nxt_slow_path(pid < 0)) { nxt_alert(task, "clone() failed for %s %E", process->name, nxt_errno); @@ -355,7 +355,7 @@ nxt_process_create(nxt_task_t *task, nxt_process_t *process) /* Parent. */ -#if (NXT_HAVE_CLONE) +#if (NXT_HAVE_LINUX_NS) nxt_debug(task, "clone(%s): %PI", process->name, pid); #else nxt_debug(task, "fork(%s): %PI", process->name, pid); @@ -781,7 +781,7 @@ nxt_process_apply_creds(nxt_task_t *task, nxt_process_t *process) cap_setid = rt->capabilities.setid; -#if (NXT_HAVE_CLONE && NXT_HAVE_CLONE_NEWUSER) +#if (NXT_HAVE_LINUX_NS && NXT_HAVE_CLONE_NEWUSER) if (!cap_setid && nxt_is_clone_flag_set(process->isolation.clone.flags, NEWUSER)) { diff --git a/src/nxt_process.h b/src/nxt_process.h index 0db68d45..1dd51521 100644 --- a/src/nxt_process.h +++ b/src/nxt_process.h @@ -7,13 +7,13 @@ #ifndef _NXT_PROCESS_H_INCLUDED_ #define _NXT_PROCESS_H_INCLUDED_ -#if (NXT_HAVE_CLONE) +#if (NXT_HAVE_LINUX_NS) #include #include #endif -#if (NXT_HAVE_CLONE) +#if (NXT_HAVE_LINUX_NS) /* * Old glibc wrapper for getpid(2) returns a cached pid invalidated only by * fork(2) calls. As we use clone(2) for container, it returns the wrong pid. @@ -100,7 +100,7 @@ typedef struct { nxt_cgroup_t cgroup; #endif -#if (NXT_HAVE_CLONE) +#if (NXT_HAVE_LINUX_NS) nxt_clone_t clone; #endif -- cgit From b0e2d9d0a185e4e2ff4bb87e399ad89119f76d1a Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Fri, 18 Nov 2022 23:53:30 +0000 Subject: Isolation: Switch to fork(2) & unshare(2) on Linux. On GitHub, @razvanphp & @hbernaciak both reported issues running the APCu PHP module under Unit. When using this module they were seeing errors like 'apcu_fetch(): Failed to acquire read lock' However when running APCu under php-fpm, everything was fine. The issue turned out to be due to our use of SYS_clone breaking the pthreads(7) API used by APCu. Even if we had been using glibc's clone(2) wrapper we would still have run into problems due to a known issue there. Essentially the problem is when using clone, glibc doesn't update the TID cache, so the child ends up having the same TID as the parent and that is used in various parts of pthreads(7) such as in the various locking primitives, so when APCu was grabbing a lock it ended up using the TID of the main unit process (rather than that of the php application processes that was grabbing the lock). So due to the above what was happening was when one of the application processes went to grab either a read or write lock, the lock was actually being attributed to the main unit process. If a process had acquired the write lock, then if a process tried to acquire a read or write lock then glibc would return EDEADLK due to detecting a deadlock situation due to thinking the process already held the write lock when in fact it didn't. It seems the right way to do this is via fork(2) and unshare(2). We already use fork(2) on other platforms. This requires a few tricks to keep the essence of the processes the same as before when using clone 1) We use the prctl(2) PR_SET_CHILD_SUBREAPER option (if its available, since Linux 3.4) to make the main unit process inherit prototype processes after a double fork(2), rather than them being reparented to 'init'. This avoids needing to ^C twice to fully exit unit when running in the foreground. It's probably also better if they maintain their parent child relationship where possible. 2) We use a double fork(2) technique on the prototype processes to ensure they themselves end up in a new PID namespace as PID 1 (when CLONE_NEWPID is being used). When using unshare(CLONE_NEWPID), the calling process is _not_ placed in the namespace (as discussed in pid_namespaces(7)). It only sets things up so that subsequent children are placed in a PID namespace. Having the prototype processes as PID 1 in the new PID namespace is probably a good thing and matches the behaviour of clone(2). Also, some isolation tests break if the prototype process is not PID 1. 3) Due to the above double fork(2) the main unit process looses track of the prototype process ID, which it needs to know. To solve this, we employ a simple pipe(2) between the main unit and prototype processes and pass the prototype grandchild PID from the parent of the second fork(2) before exiting. This needs to be done from the parent and not the grandchild, as the grandchild will see itself having a PID of 1 while the main process needs its externally visible PID. Link: Link: Closes: Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_process.c | 256 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 247 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/nxt_process.c b/src/nxt_process.c index b40eb8cf..a2980350 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -27,6 +27,19 @@ #endif +#if (NXT_HAVE_LINUX_NS) +static nxt_int_t nxt_process_pipe_timer(nxt_fd_t fd, short event); +static nxt_int_t nxt_process_check_pid_status(const nxt_fd_t *gc_pipe); +static nxt_pid_t nxt_process_recv_pid(const nxt_fd_t *pid_pipe, + const nxt_fd_t *gc_pipe); +static void nxt_process_send_pid(const nxt_fd_t *pid_pipe, nxt_pid_t pid); +static nxt_int_t nxt_process_unshare(nxt_task_t *task, nxt_process_t *process, + nxt_fd_t *pid_pipe, nxt_fd_t *gc_pipe, nxt_bool_t use_pidns); +static nxt_int_t nxt_process_init_pidns(nxt_task_t *task, + const nxt_process_t *process, nxt_fd_t *pid_pipe, nxt_fd_t *gc_pipe, + nxt_bool_t *use_pidns); +#endif + static nxt_pid_t nxt_process_create(nxt_task_t *task, nxt_process_t *process); static nxt_int_t nxt_process_do_start(nxt_task_t *task, nxt_process_t *process); static nxt_int_t nxt_process_whoami(nxt_task_t *task, nxt_process_t *process); @@ -311,6 +324,217 @@ nxt_process_child_fixup(nxt_task_t *task, nxt_process_t *process) } +#if (NXT_HAVE_LINUX_NS) + +static nxt_int_t +nxt_process_pipe_timer(nxt_fd_t fd, short event) +{ + int ret; + sigset_t mask; + struct pollfd pfd; + + static const struct timespec ts = { .tv_sec = 5 }; + + /* + * Temporarily block the signals we are handling, (except + * for SIGINT & SIGTERM) so that ppoll(2) doesn't get + * interrupted. After ppoll(2) returns, our old sigmask + * will be back in effect and any pending signals will be + * delivered. + * + * This is because while the kernel ppoll syscall updates + * the struct timespec with the time remaining if it got + * interrupted with EINTR, the glibc wrapper hides this + * from us so we have no way of knowing how long to retry + * the ppoll(2) for and if we just retry with the same + * timeout we could find ourselves in an infinite loop. + */ + pthread_sigmask(SIG_SETMASK, NULL, &mask); + sigdelset(&mask, SIGINT); + sigdelset(&mask, SIGTERM); + + pfd.fd = fd; + pfd.events = event; + + ret = ppoll(&pfd, 1, &ts, &mask); + if (ret <= 0 || (ret == 1 && pfd.revents & POLLERR)) { + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_process_check_pid_status(const nxt_fd_t *gc_pipe) +{ + int8_t status; + ssize_t ret; + + close(gc_pipe[1]); + + ret = nxt_process_pipe_timer(gc_pipe[0], POLLIN); + if (ret == NXT_OK) { + ret = read(gc_pipe[0], &status, sizeof(int8_t)); + } + + if (ret <= 0) { + status = -1; + } + + close(gc_pipe[0]); + + return status; +} + + +static nxt_pid_t +nxt_process_recv_pid(const nxt_fd_t *pid_pipe, const nxt_fd_t *gc_pipe) +{ + int8_t status; + ssize_t ret; + nxt_pid_t pid; + + close(pid_pipe[1]); + close(gc_pipe[0]); + + status = 0; + + ret = nxt_process_pipe_timer(pid_pipe[0], POLLIN); + if (ret == NXT_OK) { + ret = read(pid_pipe[0], &pid, sizeof(nxt_pid_t)); + } + + if (ret <= 0) { + status = -1; + pid = -1; + } + + write(gc_pipe[1], &status, sizeof(int8_t)); + + close(pid_pipe[0]); + close(gc_pipe[1]); + + return pid; +} + + +static void +nxt_process_send_pid(const nxt_fd_t *pid_pipe, nxt_pid_t pid) +{ + nxt_int_t ret; + + close(pid_pipe[0]); + + ret = nxt_process_pipe_timer(pid_pipe[1], POLLOUT); + if (ret == NXT_OK) { + write(pid_pipe[1], &pid, sizeof(nxt_pid_t)); + } + + close(pid_pipe[1]); +} + + +static nxt_int_t +nxt_process_unshare(nxt_task_t *task, nxt_process_t *process, + nxt_fd_t *pid_pipe, nxt_fd_t *gc_pipe, + nxt_bool_t use_pidns) +{ + int ret; + nxt_pid_t pid; + + if (process->isolation.clone.flags == 0) { + return NXT_OK; + } + + ret = unshare(process->isolation.clone.flags); + if (nxt_slow_path(ret == -1)) { + nxt_alert(task, "unshare() failed for %s %E", process->name, + nxt_errno); + + if (use_pidns) { + nxt_pipe_close(task, gc_pipe); + nxt_pipe_close(task, pid_pipe); + } + + return NXT_ERROR; + } + + if (!use_pidns) { + return NXT_OK; + } + + /* + * PID namespace requested. Employ a double fork(2) technique + * so that the prototype process will be placed into the new + * namespace and end up with PID 1 (as before with clone). + */ + pid = fork(); + if (nxt_slow_path(pid < 0)) { + nxt_alert(task, "fork() failed for %s %E", process->name, nxt_errno); + nxt_pipe_close(task, gc_pipe); + nxt_pipe_close(task, pid_pipe); + + return NXT_ERROR; + + } else if (pid > 0) { + nxt_pipe_close(task, gc_pipe); + nxt_process_send_pid(pid_pipe, pid); + + _exit(EXIT_SUCCESS); + } + + nxt_pipe_close(task, pid_pipe); + ret = nxt_process_check_pid_status(gc_pipe); + if (ret == -1) { + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_process_init_pidns(nxt_task_t *task, const nxt_process_t *process, + nxt_fd_t *pid_pipe, nxt_fd_t *gc_pipe, + nxt_bool_t *use_pidns) +{ + int ret; + + *use_pidns = 0; + +#if (NXT_HAVE_CLONE_NEWPID) + *use_pidns = nxt_is_pid_isolated(process); +#endif + + if (!*use_pidns) { + return NXT_OK; + } + + ret = nxt_pipe_create(task, pid_pipe, 0, 0); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } + + ret = nxt_pipe_create(task, gc_pipe, 0, 0); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } + +#if (NXT_HAVE_PR_SET_CHILD_SUBREAPER) + ret = prctl(PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0); + if (nxt_slow_path(ret == -1)) { + nxt_alert(task, "prctl(PR_SET_CHILD_SUBREAPER) failed for %s %E", + process->name, nxt_errno); + } +#endif + + return NXT_OK; +} + +#endif /* NXT_HAVE_LINUX_NS */ + + static nxt_pid_t nxt_process_create(nxt_task_t *task, nxt_process_t *process) { @@ -319,22 +543,31 @@ nxt_process_create(nxt_task_t *task, nxt_process_t *process) nxt_runtime_t *rt; #if (NXT_HAVE_LINUX_NS) - pid = nxt_clone(SIGCHLD | process->isolation.clone.flags); - if (nxt_slow_path(pid < 0)) { - nxt_alert(task, "clone() failed for %s %E", process->name, nxt_errno); - return pid; + nxt_fd_t pid_pipe[2], gc_pipe[2]; + nxt_bool_t use_pidns; + + ret = nxt_process_init_pidns(task, process, pid_pipe, gc_pipe, &use_pidns); + if (ret == NXT_ERROR) { + return -1; } -#else +#endif + pid = fork(); if (nxt_slow_path(pid < 0)) { nxt_alert(task, "fork() failed for %s %E", process->name, nxt_errno); return pid; } -#endif if (pid == 0) { /* Child. */ +#if (NXT_HAVE_LINUX_NS) + ret = nxt_process_unshare(task, process, pid_pipe, gc_pipe, use_pidns); + if (ret == NXT_ERROR) { + _exit(EXIT_FAILURE); + } +#endif + ret = nxt_process_child_fixup(task, process); if (nxt_slow_path(ret != NXT_OK)) { nxt_process_quit(task, 1); @@ -355,10 +588,15 @@ nxt_process_create(nxt_task_t *task, nxt_process_t *process) /* Parent. */ -#if (NXT_HAVE_LINUX_NS) - nxt_debug(task, "clone(%s): %PI", process->name, pid); -#else nxt_debug(task, "fork(%s): %PI", process->name, pid); + +#if (NXT_HAVE_LINUX_NS) + if (use_pidns) { + pid = nxt_process_recv_pid(pid_pipe, gc_pipe); + if (pid == -1) { + return pid; + } + } #endif process->pid = pid; -- cgit From 1388d311c6ead756e75cf3b7c616d054e57fe220 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Sat, 19 Nov 2022 02:27:22 +0000 Subject: Isolation: Remove nxt_clone(). Since the previous commit, this is no longer used. Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_clone.c | 14 -------------- src/nxt_clone.h | 3 --- 2 files changed, 17 deletions(-) (limited to 'src') diff --git a/src/nxt_clone.c b/src/nxt_clone.c index a98aac47..1cd70f6c 100644 --- a/src/nxt_clone.c +++ b/src/nxt_clone.c @@ -8,20 +8,6 @@ #include #include -#if (NXT_HAVE_LINUX_NS) - -pid_t -nxt_clone(nxt_int_t flags) -{ -#if defined(__s390x__) || defined(__s390__) || defined(__CRIS__) - return syscall(SYS_clone, NULL, flags); -#else - return syscall(SYS_clone, flags, NULL); -#endif -} - -#endif - #if (NXT_HAVE_CLONE_NEWUSER) diff --git a/src/nxt_clone.h b/src/nxt_clone.h index c2066ce6..6cea1bd7 100644 --- a/src/nxt_clone.h +++ b/src/nxt_clone.h @@ -33,9 +33,6 @@ typedef struct { } nxt_clone_t; -pid_t nxt_clone(nxt_int_t flags); - - #define nxt_is_clone_flag_set(flags, test) \ ((flags & CLONE_##test) == CLONE_##test) -- cgit From 3351bb4ff5cd2ef4d8f7cfb87451f021bcccb37e Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Sat, 19 Nov 2022 23:58:51 +0000 Subject: Isolation: Remove the syscall(SYS_getpid) wrapper. When using SYS_clone we used the getpid kernel system call directly via syscall(SYS_getpid) to avoid issues with cached pids. However since we are now only using fork(2) (+ unshare(2) for namespaces) we no longer need to call the kernel getpid directly as the fork(2) will ensure the cached pid is invalidated. Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_process.h | 9 --------- 1 file changed, 9 deletions(-) (limited to 'src') diff --git a/src/nxt_process.h b/src/nxt_process.h index 1dd51521..de54e383 100644 --- a/src/nxt_process.h +++ b/src/nxt_process.h @@ -13,17 +13,8 @@ #endif -#if (NXT_HAVE_LINUX_NS) -/* - * Old glibc wrapper for getpid(2) returns a cached pid invalidated only by - * fork(2) calls. As we use clone(2) for container, it returns the wrong pid. - */ -#define nxt_getpid() \ - syscall(SYS_getpid) -#else #define nxt_getpid() \ getpid() -#endif typedef pid_t nxt_pid_t; -- cgit From 96f33c0395538256ab192ea7b26203a2061c4941 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Thu, 1 Dec 2022 21:05:39 +0000 Subject: Remove the nxt_getpid() alias. Since the previous commit, nxt_getpid() is only ever aliased to getpid(2). nxt_getpid() was only used once in the code, while there are multiple direct uses of getpid(2) $ grep -r "getpid()" src/ src/nxt_unit.c: nxt_unit_pid = getpid(); src/nxt_process.c: nxt_pid = nxt_getpid(); src/nxt_process.c: nxt_pid = getpid(); src/nxt_lib.c: nxt_pid = getpid(); src/nxt_process.h:#define nxt_getpid() \ src/nxt_process.h:#define nxt_getpid() \ src/nxt_process.h: getpid() Just remove it and convert the _single_ instance of nxt_getpid() to getpid(2). Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_process.c | 2 +- src/nxt_process.h | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) (limited to 'src') diff --git a/src/nxt_process.c b/src/nxt_process.c index a2980350..fe9349e8 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -269,7 +269,7 @@ nxt_process_child_fixup(nxt_task_t *task, nxt_process_t *process) nxt_ppid = nxt_pid; - nxt_pid = nxt_getpid(); + nxt_pid = getpid(); process->pid = nxt_pid; process->isolated_pid = nxt_pid; diff --git a/src/nxt_process.h b/src/nxt_process.h index de54e383..16d6110c 100644 --- a/src/nxt_process.h +++ b/src/nxt_process.h @@ -13,9 +13,6 @@ #endif -#define nxt_getpid() \ - getpid() - typedef pid_t nxt_pid_t; -- cgit From ffa86b6edcb4ac06825557f969fb657948d8c35e Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Wed, 22 Feb 2023 16:04:53 +0000 Subject: Isolation: rootfs: Set the sticky bit on the tmp directory. When using the 'rootfs' isolation option, by default a tmpfs filesystem is mounted on tmp/. Currently this is mounted with a mode of 0777, i.e drwxrwxrwx. 3 root root 60 Feb 22 11:56 tmp however this should really have the sticky bit[0] set (as is per-normal for such directories) to prevent users from having free reign on the files contained within. What we really want is it mounted with a mode of 01777, i.e drwxrwxrwt. 3 root root 60 Feb 22 11:57 tmp [0]: To quote inode(7) "The sticky bit (S_ISVTX) on a directory means that a file in that directory can be renamed or deleted only by the owner of the file, by the owner of the directory, and by a privileged process." Reviewed-by: Liam Crilly Signed-off-by: Andrew Clayton --- src/nxt_isolation.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/nxt_isolation.c b/src/nxt_isolation.c index e43cf1f7..614d6bb5 100644 --- a/src/nxt_isolation.c +++ b/src/nxt_isolation.c @@ -652,7 +652,7 @@ nxt_isolation_set_lang_mounts(nxt_task_t *task, nxt_process_t *process, mnt->flags = (NXT_FS_FLAGS_NOSUID | NXT_FS_FLAGS_NODEV | NXT_FS_FLAGS_NOEXEC); - mnt->data = (u_char *) "size=1m,mode=777"; + mnt->data = (u_char *) "size=1m,mode=1777"; mnt->builtin = 1; mnt->deps = 0; -- cgit From 5ed6eae7188f3da17c8805a26c9e3a6f2289329a Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Thu, 23 Feb 2023 12:01:14 +0000 Subject: Set a safer umask(2) when running as a daemon. When running as a daemon. unit currently sets umask(0), i.e no umask. This is resulting in various directories being created with a mode of 0777, e.g rwxrwxrwx this is currently affecting cgroup and rootfs directories, which are being created with a mode of 0777, and when running as a daemon as there is no umask to restrict the permissions. This also affects the language modules (the umask is inherited over fork(2)) whereby unless something explicitly sets a umask, files and directories will be created with full permissions, 0666 (rw-rw-rw-)/ 0777 (rwxrwxrwx) respectively. This could be an unwitting security issue. My original idea was to just remove the umask(0) call and thus inherit the umask from the executing shell/program. However there was some concern about just inheriting whatever umask was in effect. Alex suggested that rather than simply removing the umask(0) call we change it to a value of 022 (which is a common default), which will result in directories and files with permissions at most of 0755 (rwxr-xr-x) & 0644 (rw-r--r--). If applications need some other umask set, they can (as they always have been able to) set their own umask(2). Suggested-by: Alejandro Colomar Reviewed-by: Liam Crilly Signed-off-by: Andrew Clayton --- src/nxt_process.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/nxt_process.c b/src/nxt_process.c index fe9349e8..025efe70 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -1156,10 +1156,10 @@ nxt_process_daemon(nxt_task_t *task) } /* - * Reset file mode creation mask: any access - * rights can be set on file creation. + * Set a sefe umask to give at most 755/644 permissions on + * directories/files. */ - umask(0); + umask(0022); /* Redirect STDIN and STDOUT to the "/dev/null". */ -- cgit From 78e1122a3c94a150219b4b6e1e594ae5bfdd8d68 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Sat, 4 Mar 2023 23:52:51 +0000 Subject: Router: Fix allocation of request buffer sent to application. This fixes an issue reported by @Peter2121 on GitHub. In nxt_router_prepare_msg() we create a buffer (nxt_unit_request_t *req) that gets sent to an application process that contains details about a client request. The req structure comprises various members with the final member being an array (specified as a flexible array member, with its actual length denoted by the req->fields_count member) of nxt_unit_field_t's. These structures specify the length and offset for the various request headers name/value pairs which are stored after some request metadata that is stored immediately after this array of structs as individual nul terminated strings. After this we have the body content data (if any). So it looks a little like (gdb) x /64bs 0x7f38c976e060 0x7f38c976e060: "\353\346\244\t\006" <-- First nxt_unit_field_t 0x7f38c976e066: "" 0x7f38c976e067: "" 0x7f38c976e068: "T\001" 0x7f38c976e06b: "" 0x7f38c976e06c: "Z\001" 0x7f38c976e06f: "" ... 0x7f38c976e170: "\362#\244\v$" <-- Last nxt_unit_field_t 0x7f38c976e176: "" 0x7f38c976e177: "" 0x7f38c976e178: "\342\002" 0x7f38c976e17b: "" 0x7f38c976e17c: "\352\002" 0x7f38c976e17f: "" 0x7f38c976e180: "POST" <-- Start of request metadata 0x7f38c976e185: "HTTP/1.1" 0x7f38c976e18e: "unix:" 0x7f38c976e194: "unix:/dev/shm/842.sock" 0x7f38c976e1ab: "" 0x7f38c976e1ac: "fedora" 0x7f38c976e1b3: "/842.php" 0x7f38c976e1bc: "HTTP_HOST" <-- Start of header fields 0x7f38c976e1c6: "fedora" 0x7f38c976e1cd: "HTTP_X_FORWARDED_PROTO" 0x7f38c976e1e4: "https" ... 0x7f38c976e45a: "HTTP_COOKIE" 0x7f38c976e466: "PHPSESSID=8apkg25r9s9vju3pi085i21eh4" 0x7f38c976e48b: "public_form=sended" <-- Body content Well that's how things are supposed to look! When using Unix domain sockets what we actually got looked like ... 0x7f6141f3445a: "HTTP_COOKIE" 0x7f6141f34466: "PHPSESSID=uo5b2nu9buijkc89jotbgmd60vpublic_form=sended" Here, the body content (from a POST for example) has been appended straight onto the end of the last header field value. In this case corrupting the PHP session cookie. The body content would still be found by the application as its offset into this buffer is correct. This problem was actually caused by a0327445 ("PHP: allowed to specify URLs without a trailing '/'.") which added an extra item into this request buffer specifying the port number that unit is listening on that handled this request. Unfortunately when I wrote that patch I didn't increase the size of this request buffer to accommodate it. When using normal TCP sockets we actually end up allocating more space than required for this buffer, we track the end of this buffer up to where the body content would go and so we have a few spare bytes between the nul byte of the last field header value and the start of the body content. When using Unix domain sockets, they have no associated port number and thus the port number has a length of 0 bytes, but we still write a '\0' in there using up a byte that we didn't account for, this causes us to loose the nul byte of the last header fields value causing the body data to be appended to the last header field value. The fix is simple, account for the local port length, we also add 1 to it, this covers the nul byte, even if there is no port as with Unix domain sockets. Closes: Fixes: a0327445 ("PHP: allowed to specify URLs without a trailing '/'.") Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_router.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/nxt_router.c b/src/nxt_router.c index 17f6c572..4637cc68 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -5208,6 +5208,7 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, + r->version.length + 1 + r->remote->length + 1 + r->local->length + 1 + + nxt_sockaddr_port_length(r->local) + 1 + r->server_name.length + 1 + r->target.length + 1 + (r->path->start != r->target.start ? r->path->length + 1 : 0); -- cgit From fa81d7a11a7ba31ddded934eac8e58e7c801d44a Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Tue, 28 Feb 2023 15:41:53 +0000 Subject: Perl: Fix a crash in the language module. User @bes-internal reported a Perl module crasher on GitHub. This was due to a Perl application sending back two responses, for each response we would call down into XS_NGINX__Unit__Sandbox_cb(), the first time pctx->req would point to a valid nxt_unit_request_info_t, the second time pctx->req would be NULL. Add an invalid responses check which covers this case. Closes: Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/perl/nxt_perl_psgi.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/perl/nxt_perl_psgi.c b/src/perl/nxt_perl_psgi.c index 5e8d1aee..807d1741 100644 --- a/src/perl/nxt_perl_psgi.c +++ b/src/perl/nxt_perl_psgi.c @@ -267,8 +267,11 @@ XS(XS_NGINX__Unit__Sandbox_cb) XSRETURN_EMPTY; } + pctx = CvXSUBANY(cv).any_ptr; + if (nxt_slow_path(SvOK(ST(0)) == 0 || SvROK(ST(0)) == 0 - || SvTYPE(SvRV(ST(0))) != SVt_PVAV)) + || SvTYPE(SvRV(ST(0))) != SVt_PVAV + || pctx->req == NULL)) { nxt_perl_psgi_cb_request_done(CvXSUBANY(cv).any_ptr, NXT_UNIT_ERROR); @@ -278,8 +281,6 @@ XS(XS_NGINX__Unit__Sandbox_cb) XSRETURN_EMPTY; } - pctx = CvXSUBANY(cv).any_ptr; - rc = nxt_perl_psgi_result_array(PERL_GET_CONTEXT, ST(0), pctx->req); if (nxt_slow_path(rc != NXT_UNIT_OK)) { nxt_perl_psgi_cb_request_done(CvXSUBANY(cv).any_ptr, NXT_UNIT_ERROR); -- cgit From 172ceba5b60d02cfc3acfb0feb88f415e7837092 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Mon, 13 Mar 2023 23:43:12 +0000 Subject: Router: More accurately allocate request buffer memory. In nxt_router_prepare_msg() we create a buffer (nxt_unit_request_t *req) that gets sent to an application process that contains details about a client request. This buffer was always a little larger than needed due to allocating space for the remote address _and_ port and the local address _and_ port. We also allocate space for the local port separately. ->{local,remote}->length includes the port number and ':' and also the '[]' for IPv6. E.g [2001:db8::1]:8080 ->{local,remote}->address_length represents the length of the unadorned IP address. E.g 2001:db8::1 Update the buffer size so that we only allocate what is actually needed. Suggested-by: Zhidao HONG Cc: Zhidao HONG Reviewed-by: Zhidao HONG Signed-off-by: Andrew Clayton --- src/nxt_router.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/nxt_router.c b/src/nxt_router.c index 4637cc68..17fe0418 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -5206,8 +5206,8 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, req_size = sizeof(nxt_unit_request_t) + r->method->length + 1 + r->version.length + 1 - + r->remote->length + 1 - + r->local->length + 1 + + r->remote->address_length + 1 + + r->local->address_length + 1 + nxt_sockaddr_port_length(r->local) + 1 + r->server_name.length + 1 + r->target.length + 1 -- cgit From ccaad38bc529897218a1e41c7e8704566695d1a9 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Sat, 25 Feb 2023 23:37:00 +0000 Subject: Socket: Remove Unix domain listen sockets at shutdown. If we don't remove the Unix domain listen socket file then when Unit restarts it get an error like 2023/02/25 23:10:11 [alert] 36388#36388 bind(\"unix:/tmp/unit.sock\") failed (98: Address already in use) This patch makes use of the listen_sockets array, that is already allocated in the main process but never populated, to place the Unix domain listen sockets into. At shutdown we can then loop through this array and unlink(2) any Unix domain sockets found therein. Closes: Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_main_process.c | 8 ++++++-- src/nxt_runtime.c | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 4c89121e..7548be9a 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -1182,8 +1182,9 @@ nxt_main_listening_socket(nxt_sockaddr_t *sa, nxt_listening_socket_t *ls) if (sa->u.sockaddr.sa_family == AF_UNIX && sa->u.sockaddr_un.sun_path[0] != '\0') { - char *filename; - mode_t access; + char *filename; + mode_t access; + nxt_thread_t *thr; filename = sa->u.sockaddr_un.sun_path; access = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); @@ -1194,6 +1195,9 @@ nxt_main_listening_socket(nxt_sockaddr_t *sa, nxt_listening_socket_t *ls) filename, nxt_errno); goto fail; } + + thr = nxt_thread(); + nxt_runtime_listen_socket_add(thr->runtime, sa); } #endif diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index cb5fe421..739238af 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -563,6 +563,7 @@ nxt_runtime_exit(nxt_task_t *task, void *obj, void *data) #if (NXT_HAVE_UNIX_DOMAIN) { + size_t i; nxt_sockaddr_t *sa; nxt_file_name_t *name; @@ -572,6 +573,22 @@ nxt_runtime_exit(nxt_task_t *task, void *obj, void *data) name = (nxt_file_name_t *) sa->u.sockaddr_un.sun_path; (void) nxt_file_delete(name); } + + for (i = 0; i < rt->listen_sockets->nelts; i++) { + nxt_listen_socket_t *ls; + + ls = (nxt_listen_socket_t *) rt->listen_sockets->elts + i; + sa = ls->sockaddr; + + if (sa->u.sockaddr.sa_family != AF_UNIX + || sa->u.sockaddr_un.sun_path[0] == '\0') + { + continue; + } + + name = (nxt_file_name_t *) sa->u.sockaddr_un.sun_path; + (void) nxt_file_delete(name); + } } #endif } -- cgit From 8f0192bb4cd953b7e8ca0f82f789a21fc5745220 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Thu, 9 Mar 2023 23:26:56 +0000 Subject: Remove some dormant code from nxt_process_quit(). In nxt_process_quit() there is a loop that iterates over the task->thread->runtime->listen_sockets array and closes the connections. This code has been there from the beginning $ git log --pretty=oneline -S'if (rt->listen_sockets != NULL)' e9e5ddd5a5d9ce99768833137eac2551a710becf Refactor of process management. 6f2c9acd1841ca20a1388b34aef64e9f00459090 Processes refactoring. The cycle has been renamed to the runtime. $ git log --pretty=oneline -S'if (cycle->listen_sockets != NULL) {' 6f2c9acd1841ca20a1388b34aef64e9f00459090 Processes refactoring. The cycle has been renamed to the runtime. 16cbf3c076a0aca6d47adaf3f719493674cf2363 Initial version. but never seems to have been used (AFAICT and certainly not recently, confirmed by code inspection and running pytests with a bunch of language modules enabled and the code in question was never executed) as the listen_sockets array has never been populated... until now. The previous commit now adds Unix domain sockets to this array so that they can be unlink(2)'d upon exit and reconfiguration. This has now caused this dormant code to become active as it now tries to close these sockets (from at least the prototype processes), this array is inherited via fork by other processes. The file descriptor for these sockets is set to -1 when they are put into this array. This then results in close(-1) calls which caused multiple failures in the pytests such as > assert not alerts, 'alert(s)' E AssertionError: alert(s) E assert not ['2023/03/09 23:26:14 [alert] 137673#137673 socket close(-1) failed (9: Bad file descriptor)'] I think the simplest thing is to just remove this code. Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_process.c | 21 --------------------- 1 file changed, 21 deletions(-) (limited to 'src') diff --git a/src/nxt_process.c b/src/nxt_process.c index 025efe70..c7f10769 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -1251,14 +1251,9 @@ nxt_process_close_ports(nxt_task_t *task, nxt_process_t *process) void nxt_process_quit(nxt_task_t *task, nxt_uint_t exit_status) { - nxt_uint_t n; nxt_queue_t *listen; - nxt_runtime_t *rt; nxt_queue_link_t *link, *next; nxt_listen_event_t *lev; - nxt_listen_socket_t *ls; - - rt = task->thread->runtime; nxt_debug(task, "close listen connections"); @@ -1275,21 +1270,5 @@ nxt_process_quit(nxt_task_t *task, nxt_uint_t exit_status) nxt_fd_event_close(task->thread->engine, &lev->socket); } - if (rt->listen_sockets != NULL) { - - ls = rt->listen_sockets->elts; - n = rt->listen_sockets->nelts; - - while (n != 0) { - nxt_socket_close(task, ls->socket); - ls->socket = -1; - - ls++; - n--; - } - - rt->listen_sockets->nelts = 0; - } - nxt_runtime_quit(task, exit_status); } -- cgit From 2e3e1c7e7bd5ee177b2703fa3d261fe51164426f Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Tue, 28 Feb 2023 01:59:04 +0000 Subject: Socket: Remove Unix domain listen sockets upon reconfigure. Currently when using Unix domain sockets for requests, if unit is reconfigured then it will fail if it tries to bind(2) again to a Unix domain socket with something like 2023/02/25 19:15:50 [alert] 35274#35274 bind(\"unix:/tmp/unit.sock\") failed (98: Address already in use) When closing such a socket we really need to unlink(2) it. However that presents a problem in that when running as root, while the main process runs as root and creates the socket, it's the router process, that runs as an unprivileged user, e.g nobody, that closes the socket and would thus remove it, but couldn't due to not having permission, even if the socket is mode 0666, you need write permissions on the containing directory to remove a file. There are several options to solve this, all with varying degrees of complexity and utility. 1) Give the user who the router process runs as write permission to the directory containing the listen sockets. These can then be unlink(2)'d from the router process. Simple and would work, but perhaps not the most elegant. 2) Using capabilities(7). The router process could temporarily attain the CAP_DAC_OVERRIDE capability, unlink(7) the socket, then relinquish the capability until required again. These are Linux specific (other systems may have similar mechanisms which would be extra work to support). There is also a, albeit small, window where the router process is running with elevated privileges. 3) Have the main process do the unlink(2), it is after all the process that created the socket. This is what this commit implements. We create a new port IPC message type of NXT_PORT_MSG_SOCKET_UNLINK, that is used by the router process to notify the main process about a Unix domain socket to unlink(2). Upon doing a reconfigure the router process will call nxt_router_listen_socket_release() which will close the socket, we extend this function in the case of non-abstract Unix domain sockets, so that it will send a message to the main process containing a copy of the nxt_sockaddr_t structure that will contain the filename of the socket. In the main process the handler that we have defined, nxt_main_port_socket_unlink_handler(), for this message type will run and allow us to look for the socket in question in the listen_sockets array and remove it and unlink(2) the socket. This then allows the reconfigure to work if it tries to bind(2) again to a socket that previously existed. Link: Link: Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_main_process.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ src/nxt_port.h | 3 +++ src/nxt_router.c | 41 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 87 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 7548be9a..db1cfcb9 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -48,6 +48,8 @@ static void nxt_main_process_signal_handler(nxt_task_t *task, void *obj, static void nxt_main_process_cleanup(nxt_task_t *task, nxt_process_t *process); static void nxt_main_port_socket_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +static void nxt_main_port_socket_unlink_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); static nxt_int_t nxt_main_listening_socket(nxt_sockaddr_t *sa, nxt_listening_socket_t *ls); static void nxt_main_port_modules_handler(nxt_task_t *task, @@ -587,6 +589,7 @@ static nxt_port_handlers_t nxt_main_process_port_handlers = { .remove_pid = nxt_port_remove_pid_handler, .start_process = nxt_main_start_process_handler, .socket = nxt_main_port_socket_handler, + .socket_unlink = nxt_main_port_socket_unlink_handler, .modules = nxt_main_port_modules_handler, .conf_store = nxt_main_port_conf_store_handler, #if (NXT_TLS) @@ -1214,6 +1217,49 @@ fail: } +static void +nxt_main_port_socket_unlink_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ +#if (NXT_HAVE_UNIX_DOMAIN) + size_t i; + nxt_buf_t *b; + const char *filename; + nxt_runtime_t *rt; + nxt_sockaddr_t *sa; + nxt_listen_socket_t *ls; + + b = msg->buf; + sa = (nxt_sockaddr_t *) b->mem.pos; + + filename = sa->u.sockaddr_un.sun_path; + unlink(filename); + + rt = task->thread->runtime; + + for (i = 0; i < rt->listen_sockets->nelts; i++) { + const char *name; + + ls = (nxt_listen_socket_t *) rt->listen_sockets->elts + i; + sa = ls->sockaddr; + + if (sa->u.sockaddr.sa_family != AF_UNIX + || sa->u.sockaddr_un.sun_path[0] == '\0') + { + continue; + } + + name = sa->u.sockaddr_un.sun_path; + if (strcmp(name, filename) != 0) { + continue; + } + + nxt_array_remove(rt->listen_sockets, ls); + break; + } +#endif +} + + static nxt_conf_map_t nxt_app_lang_module_map[] = { { nxt_string("type"), diff --git a/src/nxt_port.h b/src/nxt_port.h index 3a8da5ad..eba8d06f 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -16,6 +16,7 @@ struct nxt_port_handlers_s { /* Main process RPC requests. */ nxt_port_handler_t start_process; nxt_port_handler_t socket; + nxt_port_handler_t socket_unlink; nxt_port_handler_t modules; nxt_port_handler_t conf_store; nxt_port_handler_t cert_get; @@ -81,6 +82,7 @@ typedef enum { _NXT_PORT_MSG_START_PROCESS = nxt_port_handler_idx(start_process), _NXT_PORT_MSG_SOCKET = nxt_port_handler_idx(socket), + _NXT_PORT_MSG_SOCKET_UNLINK = nxt_port_handler_idx(socket_unlink), _NXT_PORT_MSG_MODULES = nxt_port_handler_idx(modules), _NXT_PORT_MSG_CONF_STORE = nxt_port_handler_idx(conf_store), _NXT_PORT_MSG_CERT_GET = nxt_port_handler_idx(cert_get), @@ -122,6 +124,7 @@ typedef enum { NXT_PORT_MSG_RPC_ERROR = nxt_msg_last(_NXT_PORT_MSG_RPC_ERROR), NXT_PORT_MSG_START_PROCESS = nxt_msg_last(_NXT_PORT_MSG_START_PROCESS), NXT_PORT_MSG_SOCKET = nxt_msg_last(_NXT_PORT_MSG_SOCKET), + NXT_PORT_MSG_SOCKET_UNLINK = nxt_msg_last(_NXT_PORT_MSG_SOCKET_UNLINK), NXT_PORT_MSG_MODULES = nxt_msg_last(_NXT_PORT_MSG_MODULES), NXT_PORT_MSG_CONF_STORE = nxt_msg_last(_NXT_PORT_MSG_CONF_STORE), NXT_PORT_MSG_CERT_GET = nxt_msg_last(_NXT_PORT_MSG_CERT_GET), diff --git a/src/nxt_router.c b/src/nxt_router.c index 17fe0418..baede83b 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -3686,6 +3686,13 @@ nxt_router_listen_socket_close(nxt_task_t *task, void *obj, void *data) static void nxt_router_listen_socket_release(nxt_task_t *task, nxt_socket_conf_t *skcf) { +#if (NXT_HAVE_UNIX_DOMAIN) + size_t size; + nxt_buf_t *b; + nxt_port_t *main_port; + nxt_runtime_t *rt; + nxt_sockaddr_t *sa; +#endif nxt_listen_socket_t *ls; nxt_thread_spinlock_t *lock; @@ -3703,10 +3710,38 @@ nxt_router_listen_socket_release(nxt_task_t *task, nxt_socket_conf_t *skcf) nxt_thread_spin_unlock(lock); - if (ls != NULL) { - nxt_socket_close(task, ls->socket); - nxt_free(ls); + if (ls == NULL) { + return; + } + + nxt_socket_close(task, ls->socket); + +#if (NXT_HAVE_UNIX_DOMAIN) + sa = ls->sockaddr; + if (sa->u.sockaddr.sa_family != AF_UNIX + || sa->u.sockaddr_un.sun_path[0] == '\0') + { + goto out_free_ls; } + + size = nxt_sockaddr_size(ls->sockaddr); + + b = nxt_buf_mem_alloc(task->thread->engine->mem_pool, size, 0); + if (b == NULL) { + goto out_free_ls; + } + + b->mem.free = nxt_cpymem(b->mem.free, ls->sockaddr, size); + + rt = task->thread->runtime; + main_port = rt->port_by_type[NXT_PROCESS_MAIN]; + + (void) nxt_port_socket_write(task, main_port, NXT_PORT_MSG_SOCKET_UNLINK, + -1, 0, 0, b); + +out_free_ls: +#endif + nxt_free(ls); } -- cgit From 7d0ceb82c71b9fc2c2884d2eeaba87fb546ef92b Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Thu, 16 Mar 2023 12:42:59 +0000 Subject: Improve an error message regarding Unix domain sockets. When starting unit, if its Unix domain control socket was already active you would get an error message like 2023/03/15 18:07:55 [alert] 53875#8669650 connect(5, unix:/tmp/control.sock) succeed, address already in use which is confusing in a couple of regards, firstly we have the classic success/failure message and secondly 'address already in use' is an actual errno value, EADDRINUSE and we didn't get an error from this connect(2). Re-word this error message for greater clarity. Reported-by: Liam Crilly Cc: Liam Crilly Signed-off-by: Andrew Clayton --- src/nxt_listen_socket.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/nxt_listen_socket.c b/src/nxt_listen_socket.c index f10abdef..d477eef1 100644 --- a/src/nxt_listen_socket.c +++ b/src/nxt_listen_socket.c @@ -161,7 +161,7 @@ nxt_listen_socket_create(nxt_task_t *task, nxt_mp_t *mp, nxt_socket_close(task, ts); if (ret == 0) { - nxt_alert(task, "connect(%d, %*s) succeed, address already in use", + nxt_alert(task, "connect(%d, %*s) socket already in use", ts, (size_t) orig_sa->length, nxt_sockaddr_start(orig_sa)); -- cgit From c18dd1f65b9eba988bb621a4b540fb6c7bda36c8 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Thu, 16 Mar 2023 21:35:01 +0000 Subject: Default PR_SET_NO_NEW_PRIVS to off. This prctl(2) option was enabled in commit 0277d8f1 ("Isolation: Fix the enablement of PR_SET_NO_NEW_PRIVS.") and this was being set by default. This prctl(2) when enabled renders (amongst other things) the set-UID and set-GID bits on executables ineffective after an execve(2). This causes an issue for applications that want to execute the sendmail(8) binary, this includes the PHP mail() function, which is usually set-GID. After some internal discussion it was decided to disable this option by default. Closes: Fixes: 0277d8f1 ("Isolation: Fix the enablement of PR_SET_NO_NEW_PRIVS.") Fixes: e2b53e16 ("Added "rootfs" feature.") Signed-off-by: Andrew Clayton --- src/nxt_isolation.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src') diff --git a/src/nxt_isolation.c b/src/nxt_isolation.c index 614d6bb5..cfa494a8 100644 --- a/src/nxt_isolation.c +++ b/src/nxt_isolation.c @@ -80,6 +80,10 @@ nxt_isolation_main_prefork(nxt_task_t *task, nxt_process_t *process, app_conf = process->data.app; cap_setid = rt->capabilities.setid; +#if (NXT_HAVE_PR_SET_NO_NEW_PRIVS) + process->isolation.new_privs = 1; +#endif + if (app_conf->isolation != NULL) { ret = nxt_isolation_set(task, app_conf->isolation, process); if (nxt_slow_path(ret != NXT_OK)) { -- cgit From 773c341d70d45a501e6ed4adb0fc6d385423a920 Mon Sep 17 00:00:00 2001 From: Alejandro Colomar Date: Tue, 10 Jan 2023 22:03:40 +0100 Subject: HTTP: rewrote while loop as for loop. This considerably simplifies the function, and will also help log the iteration in which we are, which corresponds to the route array element. Link: Link: Reviewed-by: Andrew Clayton Tested-by: Liam Crilly Reviewed-by: Zhidao Hong Signed-off-by: Alejandro Colomar --- src/nxt_http_route.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index 7081ff7e..a4dd8f4a 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -1540,21 +1540,17 @@ static nxt_http_action_t * nxt_http_route_handler(nxt_task_t *task, nxt_http_request_t *r, nxt_http_action_t *start) { + size_t i; nxt_http_route_t *route; nxt_http_action_t *action; - nxt_http_route_match_t **match, **end; route = start->u.route; - match = &route->match[0]; - end = match + route->items; - while (match < end) { - action = nxt_http_route_match(task, r, *match); + for (i = 0; i < route->items; i++) { + action = nxt_http_route_match(task, r, route->match[i]); if (action != NULL) { return action; } - - match++; } nxt_http_request_error(task, r, NXT_HTTP_NOT_FOUND); -- cgit From 0ebce31c9287cb97b626d61b62e83681b2864fe8 Mon Sep 17 00:00:00 2001 From: Alejandro Colomar Date: Thu, 26 Jan 2023 15:07:12 +0100 Subject: HTTP: added route logging. - Configuration: added "/config/settings/http/log_route". Type: bool Default: false This adds configurability to the error log. It allows enabling and disabling logs related to how the router performs selection of the routes. - HTTP: logging request line. Log level: [notice] The request line is essential to understand which logs correspond to which request when reading the logs. - HTTP: logging route that's been discarded. Log level: [info] - HTTP: logging route whose action is selected. Log level: [notice] - HTTP: logging when "fallback" action is taken. Log level: [notice] Closes: Link: Link: Suggested-by: Timo Stark Suggested-by: Mark L Wood-Patrick Suggested-by: Liam Crilly Tested-by: Liam Crilly Acked-by: Artem Konev Cc: Andrew Clayton Cc: Andrei Zeliankou Reviewed-by: Zhidao Hong Signed-off-by: Alejandro Colomar --- src/nxt_conf_validation.c | 3 +++ src/nxt_h1proto.c | 6 ++++++ src/nxt_http.h | 2 ++ src/nxt_http_route.c | 12 ++++++++++++ src/nxt_http_static.c | 6 ++++++ src/nxt_router.c | 6 ++++++ src/nxt_router.h | 2 ++ 7 files changed, 37 insertions(+) (limited to 'src') diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 537a3fb7..9169bcc9 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -344,6 +344,9 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { .type = NXT_CONF_VLDT_OBJECT, .validator = nxt_conf_vldt_object, .u.members = nxt_conf_vldt_static_members, + }, { + .name = nxt_string("log_route"), + .type = NXT_CONF_VLDT_BOOLEAN, }, NXT_CONF_VLDT_END diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 1e37273f..40b10af0 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -507,6 +507,7 @@ nxt_h1p_conn_request_init(nxt_task_t *task, void *obj, void *data) r->conf = joint; skcf = joint->socket_conf; + r->log_route = skcf->log_route; if (c->local == NULL) { c->local = skcf->sockaddr; @@ -653,6 +654,11 @@ nxt_h1p_header_process(nxt_task_t *task, nxt_h1proto_t *h1p, r->path = &h1p->parser.path; r->args = &h1p->parser.args; + if (nxt_slow_path(r->log_route)) { + nxt_log(task, NXT_LOG_NOTICE, "http request line \"%V %V %V\"", + r->method, &r->target, &r->version); + } + r->fields = h1p->parser.fields; ret = nxt_http_fields_process(r->fields, &nxt_h1p_fields_hash, r); diff --git a/src/nxt_http.h b/src/nxt_http.h index a8725d9f..ffd3f601 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -189,6 +189,8 @@ struct nxt_http_request_s { nxt_http_status_t status:16; + uint8_t log_route; /* 1 bit */ + uint8_t pass_count; /* 8 bits */ uint8_t app_target; nxt_http_protocol_t protocol:8; /* 2 bits */ diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index a4dd8f4a..f439c957 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -1548,6 +1548,18 @@ nxt_http_route_handler(nxt_task_t *task, nxt_http_request_t *r, for (i = 0; i < route->items; i++) { action = nxt_http_route_match(task, r, route->match[i]); + + if (nxt_slow_path(r->log_route)) { + uint32_t lvl = (action == NULL) ? NXT_LOG_INFO : NXT_LOG_NOTICE; + const char *sel = (action == NULL) ? "discarded" : "selected"; + + if (route->name.length == 0) { + nxt_log(task, lvl, "\"routes/%z\" %s", i, sel); + } else { + nxt_log(task, lvl, "\"routes/%V/%z\" %s", &route->name, i, sel); + } + } + if (action != NULL) { return action; } diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 68174b9d..5e44aab4 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -196,6 +196,9 @@ nxt_http_static(nxt_task_t *task, nxt_http_request_t *r, if (!nxt_str_eq(r->method, "HEAD", 4)) { if (action->fallback != NULL) { + if (nxt_slow_path(r->log_route)) { + nxt_log(task, NXT_LOG_NOTICE, "\"fallback\" taken"); + } return action->fallback; } @@ -690,6 +693,9 @@ nxt_http_static_next(nxt_task_t *task, nxt_http_request_t *r, } if (action->fallback != NULL) { + if (nxt_slow_path(r->log_route)) { + nxt_log(task, NXT_LOG_NOTICE, "\"fallback\" taken"); + } nxt_http_request_action(task, r, action->fallback); return; } diff --git a/src/nxt_router.c b/src/nxt_router.c index baede83b..992cc039 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1513,6 +1513,12 @@ static nxt_conf_map_t nxt_router_http_conf[] = { NXT_CONF_MAP_INT8, offsetof(nxt_socket_conf_t, discard_unsafe_fields), }, + + { + nxt_string("log_route"), + NXT_CONF_MAP_INT8, + offsetof(nxt_socket_conf_t, log_route), + }, }; diff --git a/src/nxt_router.h b/src/nxt_router.h index 11094960..64095b69 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -197,6 +197,8 @@ typedef struct { nxt_str_t body_temp_path; + uint8_t log_route; /* 1 bit */ + uint8_t discard_unsafe_fields; /* 1 bit */ nxt_http_forward_t *forwarded; -- cgit From 5ba79b9b524ef746bc3269520c3f6b893f39275c Mon Sep 17 00:00:00 2001 From: Alejandro Colomar Date: Mon, 27 Mar 2023 13:43:37 +0200 Subject: Renamed --libstatedir to --statedir. In BSD systems, it's usually or some other dir under that is not , so $statedir is a more generic name. See hier(7). Reported-by: Andrei Zeliankou Reported-by: Zhidao Hong Reviewed-by: Konstantin Pavlov Reviewed-by: Andrew Clayton Cc: Liam Crilly Signed-off-by: Alejandro Colomar --- src/nxt_runtime.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index 739238af..1da52da0 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -786,7 +786,7 @@ nxt_runtime_conf_init(nxt_task_t *task, nxt_runtime_t *rt) rt->pid = NXT_PID; rt->log = NXT_LOG; rt->modules = NXT_MODULESDIR; - rt->state = NXT_LIBSTATEDIR; + rt->state = NXT_STATEDIR; rt->control = NXT_CONTROL_SOCK; rt->tmp = NXT_TMPDIR; @@ -946,7 +946,7 @@ nxt_runtime_conf_read_cmd(nxt_task_t *task, nxt_runtime_t *rt) static const char no_modules[] = "option \"--modulesdir\" requires directory\n"; static const char no_state[] = - "option \"--libstatedir\" requires directory\n"; + "option \"--statedir\" requires directory\n"; static const char no_tmp[] = "option \"--tmpdir\" requires directory\n"; static const char help[] = @@ -969,8 +969,8 @@ nxt_runtime_conf_read_cmd(nxt_task_t *task, nxt_runtime_t *rt) " --modulesdir DIR set modules directory name\n" " default: \"" NXT_MODULESDIR "\"\n" "\n" - " --libstatedir DIR set state directory name\n" - " default: \"" NXT_LIBSTATEDIR "\"\n" + " --statedir DIR set state directory name\n" + " default: \"" NXT_STATEDIR "\"\n" "\n" " --tmpdir DIR set tmp directory name\n" " default: \"" NXT_TMPDIR "\"\n" @@ -1069,7 +1069,7 @@ nxt_runtime_conf_read_cmd(nxt_task_t *task, nxt_runtime_t *rt) continue; } - if (nxt_strcmp(p, "--libstatedir") == 0) { + if (nxt_strcmp(p, "--statedir") == 0) { if (*argv == NULL) { write(STDERR_FILENO, no_state, nxt_length(no_state)); return NXT_ERROR; -- cgit From 6e16d7ac5bb86140a55ea30a35c69ee0df3eff8d Mon Sep 17 00:00:00 2001 From: Alejandro Colomar Date: Wed, 22 Mar 2023 16:55:02 +0100 Subject: Auto: mirroring installation structure in build tree. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes the build tree more organized, which is good for adding new stuff. Now, it's useful for example for adding manual pages in man3/, but it may be useful in the future for example for extending the build system to run linters (e.g., clang-tidy(1), Clang analyzer, ...) on the C source code. Previously, the build tree was quite flat, and looked like this (after `./configure && make`): $ tree -I src build build ├── Makefile ├── autoconf.data ├── autoconf.err ├── echo ├── libnxt.a ├── nxt_auto_config.h ├── nxt_version.h ├── unitd └── unitd.8 1 directory, 9 files And after this patch, it looks like this: $ tree -I src build build ├── Makefile ├── autoconf.data ├── autoconf.err ├── bin │ └── echo ├── include │ ├── nxt_auto_config.h │ └── nxt_version.h ├── lib │ ├── libnxt.a │ └── unit │ └── modules ├── sbin │ └── unitd ├── share │ └── man │ └── man8 │ └── unitd.8 └── var ├── lib │ └── unit ├── log │ └── unit └── run └── unit 17 directories, 9 files It also solves one issue introduced in 5a37171f733f ("Added default values for pathnames."). Before that commit, it was possible to run unitd from the build system (`./build/unitd`). Now, since it expects files in a very specific location, that has been broken. By having a directory structure that mirrors the installation, it's possible to trick it to believe it's installed, and run it from there: $ ./configure --prefix=./build $ make $ ./build/sbin/unitd Fixes: 5a37171f733f ("Added default values for pathnames.") Reported-by: Liam Crilly Reviewed-by: Konstantin Pavlov Reviewed-by: Andrew Clayton Cc: Andrei Zeliankou Cc: Zhidao Hong Signed-off-by: Alejandro Colomar --- src/nodejs/unit-http/binding.gyp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/nodejs/unit-http/binding.gyp b/src/nodejs/unit-http/binding.gyp index 55d965bd..e41db7c4 100644 --- a/src/nodejs/unit-http/binding.gyp +++ b/src/nodejs/unit-http/binding.gyp @@ -12,7 +12,7 @@ ], 'sources': ["unit.cpp", "addon.cpp"], 'include_dirs': [ - " Date: Tue, 14 Mar 2023 17:21:36 +0000 Subject: Remove a useless assignment in nxt_fs_mkdir_all(). This was reported by the 'Clang Static Analyzer' as a 'dead nested assignment'. We set end outside the loop but the first time we use it is to assign it in the loop (not used anywhere else). Further cleanup could be to reduce the scope of end by moving its declaration inside the loop. Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_fs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/nxt_fs.c b/src/nxt_fs.c index a467da98..e10c5bcb 100644 --- a/src/nxt_fs.c +++ b/src/nxt_fs.c @@ -20,7 +20,7 @@ nxt_fs_mkdir_all(const u_char *dir, mode_t mode) nxt_assert(dirlen < PATH_MAX && dirlen > 1 && dir[0] == '/'); dst = path; - start = end = (char *) dir; + start = (char *) dir; while (*start != '\0') { if (*start == '/') { -- cgit From 8a9e078e5427e2b72e178740f25cbcebb780b225 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Tue, 14 Mar 2023 17:31:48 +0000 Subject: Prevent a possible NULL de-reference in nxt_job_create(). We allocate 'job' we then have a check if it's not NULL and do stuff with it, but then we accessed it outside this check. Simply return if job is NULL. Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_job.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/nxt_job.c b/src/nxt_job.c index 995fd89b..56073953 100644 --- a/src/nxt_job.c +++ b/src/nxt_job.c @@ -32,12 +32,14 @@ nxt_job_create(nxt_mp_t *mp, size_t size) cache_size = size; } - if (nxt_fast_path(job != NULL)) { - job->cache_size = (uint16_t) cache_size; - job->mem_pool = mp; - nxt_job_set_name(job, "job"); + if (nxt_slow_path(job == NULL)) { + return NULL; } + job->cache_size = (uint16_t) cache_size; + job->mem_pool = mp; + nxt_job_set_name(job, "job"); + /* Allow safe nxt_queue_remove() in nxt_job_destroy(). */ nxt_queue_self(&job->link); -- cgit From 485205712433611ee9939a869b4f734f824a177a Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Wed, 15 Mar 2023 03:48:48 +0000 Subject: Remove a useless assignment in nxt_mem_zone_alloc_pages(). This was reported by the 'Clang Static Analyzer' as a 'dead nested assignment'. We assign prev_size then check if it's != 0 and if true we then set prev_pages to page_size right shifted by two at the same time setting prev_size to be right shifted by two (>>=), however page_size is never used again so no need to set it here. Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_mem_zone.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/nxt_mem_zone.c b/src/nxt_mem_zone.c index f8ab09d9..a3ba3700 100644 --- a/src/nxt_mem_zone.c +++ b/src/nxt_mem_zone.c @@ -672,7 +672,7 @@ nxt_mem_zone_alloc_pages(nxt_mem_zone_t *zone, size_t alignment, uint32_t pages) prev_size = p - (u_char *) block; if (prev_size != 0) { - prev_pages = prev_size >>= zone->page_size_shift; + prev_pages = prev_size >> zone->page_size_shift; node_pages -= prev_pages; block->size = prev_pages; -- cgit From edbc43558d40768d91b378205c2d52bd7ba9d00a Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Fri, 31 Mar 2023 14:01:43 +0100 Subject: PHP: Make the filter_input() function work. On GitHub, @jamesRUS52 reported that the PHP filter_input()[0] function would just return NULL. To enable this function we need to run the variables through the sapi_module.input_filter() function when we call php_register_variable_safe(). In PHP versions prior to 7.0.0, input_filter() takes 'len' as an unsigned int, while later versions take it as a size_t. Now, with this commit and the following PHP you get $ curl 'http://localhost:8080/854.php?get=foo<>' string(3) "::1" string(18) "/854.php?get=foo<>" string(13) "foo<>" [0]: Tested-by: Closes: Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_php_sapi.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 32a13a70..ba000fc0 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -1532,14 +1532,23 @@ static void nxt_php_set_sptr(nxt_unit_request_info_t *req, const char *name, nxt_unit_sptr_t *v, uint32_t len, zval *track_vars_array TSRMLS_DC) { - char *str; + char *str; +#if NXT_PHP7 + size_t new_len; +#else + unsigned int new_len; +#endif str = nxt_unit_sptr_get(v); nxt_unit_req_debug(req, "php: register %s='%.*s'", name, (int) len, str); - php_register_variable_safe((char *) name, str, len, - track_vars_array TSRMLS_CC); + if (sapi_module.input_filter(PARSE_SERVER, (char *) name, &str, len, + &new_len TSRMLS_CC)) + { + php_register_variable_safe((char *) name, str, new_len, + track_vars_array TSRMLS_CC); + } } -- cgit From 8b8952930c8462b5c56cdaf14e4a1c7c8b0c203d Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Thu, 30 Mar 2023 05:44:18 +0100 Subject: Add nxt_file_stdout(). This is analogous to the nxt_file_stderr() function and will be used in a subsequent commit. This function redirects stdout to a given file descriptor. Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_file.c | 19 +++++++++++++++++++ src/nxt_file.h | 1 + 2 files changed, 20 insertions(+) (limited to 'src') diff --git a/src/nxt_file.c b/src/nxt_file.c index a3fcda76..6f1a93e4 100644 --- a/src/nxt_file.c +++ b/src/nxt_file.c @@ -563,6 +563,25 @@ nxt_file_redirect(nxt_file_t *file, nxt_fd_t fd) } +/* nxt_file_stdout() redirects the stdout descriptor to the file. */ + +nxt_int_t +nxt_file_stdout(nxt_file_t *file) +{ + nxt_thread_log_debug("dup2(%FD, %FD, \"%FN\")", + file->fd, STDOUT_FILENO, file->name); + + if (dup2(file->fd, STDOUT_FILENO) != -1) { + return NXT_OK; + } + + nxt_thread_log_alert("dup2(%FD, %FD, \"%FN\") failed %E", + file->fd, STDOUT_FILENO, file->name, nxt_errno); + + return NXT_ERROR; +} + + /* nxt_file_stderr() redirects the stderr descriptor to the file. */ nxt_int_t diff --git a/src/nxt_file.h b/src/nxt_file.h index 945717b3..97636db6 100644 --- a/src/nxt_file.h +++ b/src/nxt_file.h @@ -191,6 +191,7 @@ NXT_EXPORT FILE *nxt_file_fopen(nxt_task_t *task, const char *pathname, NXT_EXPORT void nxt_file_fclose(nxt_task_t *task, FILE *fp); NXT_EXPORT nxt_int_t nxt_file_redirect(nxt_file_t *file, nxt_fd_t fd); +NXT_EXPORT nxt_int_t nxt_file_stdout(nxt_file_t *file); NXT_EXPORT nxt_int_t nxt_file_stderr(nxt_file_t *file); NXT_EXPORT nxt_int_t nxt_stderr_start(void); -- cgit From 45c45eaeb4443ff4af9cc49e716bbd9d65596b02 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Thu, 30 Mar 2023 05:53:13 +0100 Subject: Add per-application logging. Currently when running in the foreground, unit application processes will send stdout to the current TTY and stderr to the unit log file. That behaviour won't change. When running as a daemon, unit application processes will send stdout to /dev/null and stderr to the unit log file. This commit allows to alter the latter case of unit running as a daemon, by allowing applications to redirect stdout and/or stderr to specific log files. This is done via two new application options, 'stdout' & 'stderr', e.g "applications": { "myapp": { ... "stdout": "/path/to/log/unit/app/stdout.log", "stderr": "/path/to/log/unit/app/stderr.log" } } These log files are created by the application processes themselves and thus the log directories need to be writable by the user (and or group) of the application processes. E.g $ sudo mkdir -p /path/to/log/unit/app $ sudo chown APP_USER /path/to/log/unit/app These need to be setup before starting unit with the above config. Currently these log files do not participate in log-file rotation (SIGUSR1), that may change in a future commit. In the meantime these logs can be rotated using the traditional copy/truncate method. NOTE: You may or may not see stuff printed to stdout as stdout was traditionally used by CGI applications to communicate with the webserver. Closes: Closes: Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_application.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++ src/nxt_application.h | 4 ++++ src/nxt_conf_validation.c | 6 ++++++ src/nxt_main_process.c | 12 +++++++++++ src/nxt_process.c | 6 ++++++ 5 files changed, 81 insertions(+) (limited to 'src') diff --git a/src/nxt_application.c b/src/nxt_application.c index 786c768b..ffa8eb53 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -933,6 +933,59 @@ nxt_app_set_environment(nxt_conf_value_t *environment) } +nxt_int_t +nxt_app_set_logs(void) +{ + nxt_int_t ret; + nxt_file_t file; + nxt_task_t *task; + nxt_thread_t *thr; + nxt_process_t *process; + nxt_runtime_t *rt; + nxt_common_app_conf_t *app_conf; + + thr = nxt_thread(); + + task = thr->task; + + rt = task->thread->runtime; + if (!rt->daemon) { + return NXT_OK; + } + + process = rt->port_by_type[NXT_PROCESS_PROTOTYPE]->process; + app_conf = process->data.app; + + if (app_conf->stdout_log != NULL) { + nxt_memzero(&file, sizeof(nxt_file_t)); + file.log_level = 1; + file.name = (u_char *) app_conf->stdout_log; + ret = nxt_file_open(task, &file, O_WRONLY | O_APPEND, O_CREAT, 0666); + if (ret == NXT_ERROR) { + return NXT_ERROR; + } + + nxt_file_stdout(&file); + nxt_file_close(task, &file); + } + + if (app_conf->stderr_log != NULL) { + nxt_memzero(&file, sizeof(nxt_file_t)); + file.log_level = 1; + file.name = (u_char *) app_conf->stderr_log; + ret = nxt_file_open(task, &file, O_WRONLY | O_APPEND, O_CREAT, 0666); + if (ret == NXT_ERROR) { + return NXT_ERROR; + } + + nxt_file_stderr(&file); + nxt_file_close(task, &file); + } + + return NXT_OK; +} + + static u_char * nxt_cstr_dup(nxt_mp_t *mp, u_char *dst, u_char *src) { diff --git a/src/nxt_application.h b/src/nxt_application.h index 4d624448..2675e6a0 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -92,6 +92,9 @@ struct nxt_common_app_conf_s { nxt_str_t user; nxt_str_t group; + char *stdout_log; + char *stderr_log; + char *working_directory; nxt_conf_value_t *environment; @@ -141,5 +144,6 @@ extern nxt_app_module_t nxt_external_module; NXT_EXPORT nxt_int_t nxt_unit_default_init(nxt_task_t *task, nxt_unit_init_t *init, nxt_common_app_conf_t *conf); +NXT_EXPORT nxt_int_t nxt_app_set_logs(void); #endif /* _NXT_APPLICATION_H_INCLIDED_ */ diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 9169bcc9..6d798a81 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -1047,6 +1047,12 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_common_members[] = { .type = NXT_CONF_VLDT_OBJECT, .validator = nxt_conf_vldt_isolation, .u.members = nxt_conf_vldt_app_isolation_members, + }, { + .name = nxt_string("stdout"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("stderr"), + .type = NXT_CONF_VLDT_STRING, }, NXT_CONF_VLDT_END diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index db1cfcb9..ae62aff4 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -120,6 +120,18 @@ static nxt_conf_map_t nxt_common_app_conf[] = { offsetof(nxt_common_app_conf_t, group), }, + { + nxt_string("stdout"), + NXT_CONF_MAP_CSTRZ, + offsetof(nxt_common_app_conf_t, stdout_log), + }, + + { + nxt_string("stderr"), + NXT_CONF_MAP_CSTRZ, + offsetof(nxt_common_app_conf_t, stderr_log), + }, + { nxt_string("working_directory"), NXT_CONF_MAP_CSTRZ, diff --git a/src/nxt_process.c b/src/nxt_process.c index c7f10769..ce2de774 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -5,6 +5,8 @@ */ #include + +#include #include #if (NXT_HAVE_LINUX_NS) @@ -651,6 +653,10 @@ nxt_process_setup(nxt_task_t *task, nxt_process_t *process) thread = task->thread; rt = thread->runtime; + if (process->parent_port == rt->port_by_type[NXT_PROCESS_PROTOTYPE]) { + nxt_app_set_logs(); + } + nxt_random_init(&thread->random); rt->type = init->type; -- cgit From fcff55acb61be0b9818075e8fb9d3cc80dd97b31 Mon Sep 17 00:00:00 2001 From: Alejandro Colomar Date: Fri, 31 Mar 2023 14:08:41 +0200 Subject: HTTP: optimizing $request_line. Don't reconstruct a new string for the $request_line from the parsed method, target, and HTTP version, but rather keep a pointer to the original memory where the request line was received. This will be necessary for implementing URI rewrites, since we want to log the original request line, and not one constructed from the rewritten target. This implementation changes behavior (only for invalid requests) in the following way: Previous behavior was to log as many tokens from the request line as were parsed validly, thus: Request -> access log ; error log "GET / HTTP/1.1" -> "GET / HTTP/1.1" OK ; = "GET / HTTP/1.1" -> "GET / HTTP/1.1" [1] ; = "GET / HTTP/2.1" -> "GET / HTTP/2.1" OK ; = "GET / HTTP/1." -> "GET / HTTP/1." [2] ; "GET / HTTP/1. [null]" "GET / food" -> "GET / food" [2] ; "GET / food [null]" "GET / / HTTP/1.1" -> "GET / / HTTP/1.1" [2] ; = "GET / / HTTP/1.1" -> "GET / / HTTP/1.1" [2] ; = "GET food HTTP/1.1" -> "GET" ; "GET [null] [null]" "OPTIONS * HTTP/1.1" -> "OPTIONS" [3] ; "OPTIONS [null] [null]" "FOOBAR baz HTTP/1.1"-> "FOOBAR" ; "FOOBAR [null] [null]" "FOOBAR / HTTP/1.1" -> "FOOBAR / HTTP/1.1" ; = "get / HTTP/1.1" -> "-" ; " [null] [null]" "" -> "-" ; " [null] [null]" This behavior was rather inconsistent. We have several options to go forward with this patch: - NGINX behavior. Log the entire request line, up to '\r' | '\n', even if it was invalid. This is the most informative alternative. However, RFC-complying requests will probably not send invalid requests. This information would be interesting to users where debugging requests constructed manually via netcat(1) or a similar tool, or maybe for debugging a client, are important. It might be interesting to support this in the future if our users are interested; for now, since this approach requires looping over invalid requests twice, that's an overhead that we better avoid. - Previous Unit behavior This is relatively fast (almost as fast as the next alternative, the one we chose), but the implementation is ugly, in that we need to perform the same operation in many places around the code. If we want performance, probably the next alternative is better; if we want to be informative, then the first one is better (maybe in combination with the third one too). - Chosen behavior Only logging request lines when the request is valid. For any invalid request, or even unsupported ones, the request line will be logged as "-". Thus: Request -> access log [4] "GET / HTTP/1.1" -> "GET / HTTP/1.1" OK "GET / HTTP/1.1" -> "GET / HTTP/1.1" [1] "GET / HTTP/2.1" -> "-" [3] "GET / HTTP/1." -> "-" "GET / food" -> "-" "GET / / HTTP/1.1" -> "GET / / HTTP/1.1" [2] "GET / / HTTP/1.1" -> "GET / / HTTP/1.1" [2] "GET food HTTP/1.1" -> "-" "OPTIONS * HTTP/1.1" -> "-" "FOOBAR baz HTTP/1.1"-> "-" "FOOBAR / HTTP/1.1" -> "FOOBAR / HTTP/1.1" "get / HTTP/1.1" -> "-" "" -> "-" This is less informative than previous behavior, but considering how inconsistent it was, and that RFC-complying agents will probably not send us such requests, we're ready to lose that information in the log. This is of course the fastest and simplest implementation we can get. We've chosen to implement this alternative in this patch. Since we modified the behavior, this patch also changes the affected tests. [1]: Multiple successive spaces as a token delimiter is allowed by the RFC, but it is discouraged, and considered a security risk. It is currently supported by Unit, but we will probably drop support for it in the future. [2]: Unit currently supports spaces in the request-target. This is a violation of the relevant RFC (linked below), and will be fixed in the future, and consider those targets as invalid, returning a 400 (Bad Request), and thus the log lines with the previous inconsistent behavior would be changed. [3]: Not yet supported. [4]: In the error log, regarding the "log_routes" conditional logging of the request line, we only need to log the request line if it was valid. It doesn't make sense to log "" or "-" in case that the request was invalid, since this is only useful for understanding decisions of the router. In this case, the access log is more appropriate, which shows that the request was invalid, and a 400 was returned. When the request line is valid, it is printed in the error log exactly as in the access log. Link: Suggested-by: Liam Crilly Reviewed-by: Zhidao Hong Cc: Timo Stark Cc: Andrei Zeliankou Cc: Andrew Clayton Cc: Artem Konev Signed-off-by: Alejandro Colomar --- src/nxt_h1proto.c | 14 +++++++++----- src/nxt_http.h | 1 + src/nxt_http_parse.c | 14 ++++++++------ src/nxt_http_parse.h | 1 + src/nxt_http_variables.c | 31 +------------------------------ 5 files changed, 20 insertions(+), 41 deletions(-) (limited to 'src') diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 40b10af0..df1f82f9 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -577,6 +577,15 @@ nxt_h1p_conn_request_header_parse(nxt_task_t *task, void *obj, void *data) */ h1p->keepalive = (h1p->parser.version.s.minor != '0'); + r->request_line.start = h1p->parser.method.start; + r->request_line.length = h1p->parser.request_line_end + - r->request_line.start; + + if (nxt_slow_path(r->log_route)) { + nxt_log(task, NXT_LOG_NOTICE, "http request line \"%V\"", + &r->request_line); + } + ret = nxt_h1p_header_process(task, h1p, r); if (nxt_fast_path(ret == NXT_OK)) { @@ -654,11 +663,6 @@ nxt_h1p_header_process(nxt_task_t *task, nxt_h1proto_t *h1p, r->path = &h1p->parser.path; r->args = &h1p->parser.args; - if (nxt_slow_path(r->log_route)) { - nxt_log(task, NXT_LOG_NOTICE, "http request line \"%V %V %V\"", - r->method, &r->target, &r->version); - } - r->fields = h1p->parser.fields; ret = nxt_http_fields_process(r->fields, &nxt_h1p_fields_hash, r); diff --git a/src/nxt_http.h b/src/nxt_http.h index ffd3f601..d7bbaf02 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -144,6 +144,7 @@ struct nxt_http_request_s { nxt_str_t host; nxt_str_t server_name; + nxt_str_t request_line; nxt_str_t target; nxt_str_t version; nxt_str_t *method; diff --git a/src/nxt_http_parse.c b/src/nxt_http_parse.c index f39d8f67..439993df 100644 --- a/src/nxt_http_parse.c +++ b/src/nxt_http_parse.c @@ -417,23 +417,25 @@ space_after_target: { rp->version.ui64 = ver.ui64; - if (nxt_fast_path(p[9] == '\r')) { - p += 10; + p += 9; + if (nxt_fast_path(*p == '\r')) { - if (nxt_slow_path(p == end)) { + if (nxt_slow_path(p + 1 == end)) { return NXT_AGAIN; } - if (nxt_slow_path(*p != '\n')) { + if (nxt_slow_path(p[1] != '\n')) { return NXT_HTTP_PARSE_INVALID; } - *pos = p + 1; + *pos = p + 2; } else { - *pos = p + 10; + *pos = p + 1; } + rp->request_line_end = p; + if (rp->complex_target != 0 #if 0 || rp->quoted_target != 0 diff --git a/src/nxt_http_parse.h b/src/nxt_http_parse.h index 2b714464..88b10675 100644 --- a/src/nxt_http_parse.h +++ b/src/nxt_http_parse.h @@ -41,6 +41,7 @@ struct nxt_http_request_parse_s { u_char *target_start; u_char *target_end; + u_char *request_line_end; nxt_str_t path; nxt_str_t args; diff --git a/src/nxt_http_variables.c b/src/nxt_http_variables.c index fa0244db..b73d9151 100644 --- a/src/nxt_http_variables.c +++ b/src/nxt_http_variables.c @@ -273,40 +273,11 @@ static nxt_int_t nxt_http_var_request_line(nxt_task_t *task, nxt_str_t *str, void *ctx, uint16_t field) { - size_t length; - u_char *p, *start; nxt_http_request_t *r; r = ctx; - length = r->method->length + 1 + r->target.length + 1 + r->version.length; - - start = nxt_mp_nget(r->mem_pool, length); - if (nxt_slow_path(start == NULL)) { - return NXT_ERROR; - } - - p = start; - - if (r->method->length != 0) { - p = nxt_cpymem(p, r->method->start, r->method->length); - - if (r->target.length != 0) { - *p++ = ' '; - p = nxt_cpymem(p, r->target.start, r->target.length); - - if (r->version.length != 0) { - *p++ = ' '; - p = nxt_cpymem(p, r->version.start, r->version.length); - } - } - - } else { - *p++ = '-'; - } - - str->start = start; - str->length = p - start; + *str = r->request_line; return NXT_OK; } -- cgit From b9177d36e71a9f62198b00fa40f277c06d2264bb Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Mon, 27 Mar 2023 19:28:54 +0100 Subject: Remove a bunch of dead code. This removes a bunch of unused files that would have been touched by subsequent commits that switch to using nxt_bool_t (AKA unit6_t) in structures. In auto/sources we have NXT_LIB_SRC0=" \ src/nxt_buf_filter.c \ src/nxt_job_file.c \ src/nxt_stream_module.c \ src/nxt_stream_source.c \ src/nxt_upstream_source.c \ src/nxt_http_source.c \ src/nxt_fastcgi_source.c \ src/nxt_fastcgi_record_parse.c \ \ src/nxt_mem_pool_cleanup.h \ src/nxt_mem_pool_cleanup.c \ " None of these seem to actually be used anywhere (other than within themselves). That variable is _not_ referenced anywhere else. Also remove the unused related header files: src/nxt_buf_filter.h, src/nxt_fastcgi_source.h, src/nxt_http_source.h, src/nxt_job_file.h, src/nxt_stream_source.h and src/nxt_upstream_source.h Also, these files do not seem to be used, no mention under auto/ or build/ src/nxt_file_cache.c src/nxt_cache.c src/nxt_job_file_cache.c src/nxt_cache.h is #included in src/nxt_main.h, but AFAICT is not actually used. With all the above removed $ ./configure --openssl --debug --tests && make -j && make -j tests && make libnxt all builds. Buildbot passes. NOTE: You may need to do a 'make clean' before the next build attempt. Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_buf_filter.c | 449 ------------------------ src/nxt_buf_filter.h | 115 ------- src/nxt_cache.c | 642 ----------------------------------- src/nxt_cache.h | 122 ------- src/nxt_fastcgi_record_parse.c | 307 ----------------- src/nxt_fastcgi_source.c | 750 ----------------------------------------- src/nxt_fastcgi_source.h | 93 ----- src/nxt_file_cache.c | 508 ---------------------------- src/nxt_http_source.c | 629 ---------------------------------- src/nxt_http_source.h | 47 --- src/nxt_job_file.c | 302 ----------------- src/nxt_job_file.h | 74 ---- src/nxt_job_file_cache.c | 42 --- src/nxt_main.h | 4 - src/nxt_mem_pool_cleanup.c | 39 --- src/nxt_mem_pool_cleanup.h | 15 - src/nxt_stream_module.c | 131 ------- src/nxt_stream_source.c | 480 -------------------------- src/nxt_stream_source.h | 32 -- src/nxt_upstream_source.c | 71 ---- src/nxt_upstream_source.h | 83 ----- 21 files changed, 4935 deletions(-) delete mode 100644 src/nxt_buf_filter.c delete mode 100644 src/nxt_buf_filter.h delete mode 100644 src/nxt_cache.c delete mode 100644 src/nxt_cache.h delete mode 100644 src/nxt_fastcgi_record_parse.c delete mode 100644 src/nxt_fastcgi_source.c delete mode 100644 src/nxt_fastcgi_source.h delete mode 100644 src/nxt_file_cache.c delete mode 100644 src/nxt_http_source.c delete mode 100644 src/nxt_http_source.h delete mode 100644 src/nxt_job_file.c delete mode 100644 src/nxt_job_file.h delete mode 100644 src/nxt_job_file_cache.c delete mode 100644 src/nxt_mem_pool_cleanup.c delete mode 100644 src/nxt_mem_pool_cleanup.h delete mode 100644 src/nxt_stream_module.c delete mode 100644 src/nxt_stream_source.c delete mode 100644 src/nxt_stream_source.h delete mode 100644 src/nxt_upstream_source.c delete mode 100644 src/nxt_upstream_source.h (limited to 'src') diff --git a/src/nxt_buf_filter.c b/src/nxt_buf_filter.c deleted file mode 100644 index 83e5baa9..00000000 --- a/src/nxt_buf_filter.c +++ /dev/null @@ -1,449 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -static nxt_int_t nxt_buf_filter_nobuf(nxt_buf_filter_t *f); -nxt_inline void nxt_buf_filter_next(nxt_buf_filter_t *f); -static void nxt_buf_filter_file_read_start(nxt_task_t *task, - nxt_buf_filter_t *f); -static void nxt_buf_filter_file_read(nxt_task_t *task, nxt_buf_filter_t *f); -static void nxt_buf_filter_file_job_completion(nxt_task_t *task, void *obj, - void *data); -static void nxt_buf_filter_buf_completion(nxt_task_t *task, void *obj, - void *data); -static void nxt_buf_filter_file_read_error(nxt_task_t *task, void *obj, - void *data); - - -void -nxt_buf_filter_add(nxt_task_t *task, nxt_buf_filter_t *f, nxt_buf_t *b) -{ - nxt_buf_chain_add(&f->input, b); - - nxt_buf_filter(task, f, NULL); -} - - -void -nxt_buf_filter(nxt_task_t *task, void *obj, void *data) -{ - nxt_int_t ret; - nxt_buf_t *b; - nxt_buf_filter_t *f; - - f = obj; - - nxt_debug(task, "buf filter"); - - if (f->done) { - return; - } - - f->queued = 0; - - for ( ;; ) { - /* - * f->input is a chain of original incoming buffers: memory, - * mapped, file, and sync buffers; - * f->current is a currently processed memory buffer or a chain - * of memory/file or mapped/file buffers which are read of - * or populated from file; - * f->output is a chain of output buffers; - * f->last is the last output buffer in the chain. - */ - - b = f->current; - - nxt_debug(task, "buf filter current: %p", b); - - if (b == NULL) { - - if (f->reading) { - return; - } - - b = f->input; - - nxt_debug(task, "buf filter input: %p", b); - - if (b == NULL) { - /* - * The end of the input chain, pass - * the output chain to the next filter. - */ - nxt_buf_filter_next(f); - - return; - } - - if (nxt_buf_is_mem(b)) { - - f->current = b; - f->input = b->next; - b->next = NULL; - - } else if (nxt_buf_is_file(b)) { - - if (f->run->filter_ready(f) != NXT_OK) { - nxt_buf_filter_next(f); - } - - nxt_buf_filter_file_read_start(task, f); - return; - } - } - - if (nxt_buf_is_sync(b)) { - - ret = NXT_OK; - f->current = b; - f->input = b->next; - b->next = NULL; - - if (nxt_buf_is_nobuf(b)) { - ret = f->run->filter_sync_nobuf(f); - - } else if (nxt_buf_is_flush(b)) { - ret = f->run->filter_sync_flush(f); - - } else if (nxt_buf_is_last(b)) { - ret = f->run->filter_sync_last(f); - - f->done = (ret == NXT_OK); - } - - if (nxt_fast_path(ret == NXT_OK)) { - continue; - } - - if (nxt_slow_path(ret == NXT_ERROR)) { - goto fail; - } - - /* ret == NXT_AGAIN: No filter internal buffers available. */ - goto nobuf; - } - - ret = f->run->filter_process(f); - - if (nxt_fast_path(ret == NXT_OK)) { - b = f->current; - /* - * A filter may just move f->current to f->output - * and then set f->current to NULL. - */ - if (b != NULL && b->mem.pos == b->mem.free) { - f->current = b->next; - nxt_thread_work_queue_add(task->thread, f->work_queue, - b->completion_handler, - task, b, b->parent); - } - - continue; - } - - if (nxt_slow_path(ret == NXT_ERROR)) { - goto fail; - } - - /* ret == NXT_AGAIN: No filter internal buffers available. */ - goto nobuf; - } - -nobuf: - - /* ret == NXT_AGAIN: No filter internal buffers available. */ - - if (nxt_buf_filter_nobuf(f) == NXT_OK) { - return; - } - -fail: - - nxt_thread_work_queue_add(task->thread, f->work_queue, f->run->filter_error, - task, f, f->data); -} - - -static nxt_int_t -nxt_buf_filter_nobuf(nxt_buf_filter_t *f) -{ - nxt_buf_t *b; - - nxt_thread_log_debug("buf filter nobuf"); - - b = nxt_buf_sync_alloc(f->mem_pool, NXT_BUF_SYNC_NOBUF); - - if (nxt_fast_path(b != NULL)) { - - nxt_buf_chain_add(&f->output, b); - f->last = NULL; - - f->run->filter_next(f); - - f->output = NULL; - - return NXT_OK; - } - - return NXT_ERROR; -} - - -nxt_inline void -nxt_buf_filter_next(nxt_buf_filter_t *f) -{ - if (f->output != NULL) { - f->last = NULL; - - f->run->filter_next(f); - f->output = NULL; - } -} - - -void -nxt_buf_filter_enqueue(nxt_task_t *task, nxt_buf_filter_t *f) -{ - nxt_debug(task, "buf filter enqueue: %d", f->queued); - - if (!f->queued && !f->done) { - f->queued = 1; - nxt_thread_work_queue_add(task->thread, f->work_queue, nxt_buf_filter, - task, f, NULL); - } -} - - -static void -nxt_buf_filter_file_read_start(nxt_task_t *task, nxt_buf_filter_t *f) -{ - nxt_job_file_t *jbf; - nxt_buf_filter_file_t *ff; - - ff = f->run->job_file_create(f); - - if (nxt_slow_path(ff == NULL)) { - nxt_thread_work_queue_add(task->thread, f->work_queue, - f->run->filter_error, - task, f, f->data); - return; - } - - f->filter_file = ff; - - jbf = &ff->job_file; - jbf->file = *f->input->file; - - jbf->ready_handler = nxt_buf_filter_file_job_completion; - jbf->error_handler = nxt_buf_filter_file_read_error; - - nxt_job_set_name(&jbf->job, "buf filter job file"); - - f->reading = 1; - - nxt_buf_filter_file_read(task, f); -} - - -static void -nxt_buf_filter_file_read(nxt_task_t *task, nxt_buf_filter_t *f) -{ - nxt_int_t ret; - nxt_off_t size; - nxt_buf_t *b; - nxt_buf_filter_file_t *ff; - - ff = f->filter_file; - - if (ff->job_file.buffer != NULL) { - /* File is now being read. */ - return; - } - - size = f->input->file_end - f->input->file_pos; - - if (size > (nxt_off_t) NXT_SIZE_T_MAX) { - /* - * Small size value is a hint for buffer pool allocation - * size, but if size of the size_t type is lesser than size - * of the nxt_off_t type, the large size value may be truncated, - * so use a default buffer pool allocation size. - */ - size = 0; - } - - if (f->mmap) { - ret = nxt_buf_pool_mmap_alloc(&ff->buffers, (size_t) size); - - } else { - ret = nxt_buf_pool_file_alloc(&ff->buffers, (size_t) size); - } - - if (nxt_fast_path(ret == NXT_OK)) { - b = ff->buffers.current; - - b->file_pos = f->input->file_pos; - b->file_end = f->input->file_pos; - b->file = f->input->file; - - ff->job_file.buffer = b; - ff->job_file.offset = f->input->file_pos; - - f->run->job_file_retain(f); - - nxt_job_file_read(task, &ff->job_file.job); - return; - } - - if (nxt_fast_path(ret != NXT_ERROR)) { - - /* ret == NXT_AGAIN: No buffers available. */ - - if (f->buffering) { - f->buffering = 0; - - if (nxt_fast_path(f->run->filter_flush(f) != NXT_ERROR)) { - return; - } - - } else if (nxt_fast_path(nxt_buf_filter_nobuf(f) == NXT_OK)) { - return; - } - } - - nxt_thread_work_queue_add(task->thread, f->work_queue, f->run->filter_error, - task, f, f->data); -} - - -typedef struct { - nxt_buf_filter_t *filter; - nxt_buf_t *buf; -} nxt_buf_filter_ctx_t; - - -static void -nxt_buf_filter_file_job_completion(nxt_task_t *task, void *obj, void *data) -{ - nxt_buf_t *b; - nxt_bool_t done; - nxt_job_file_t *jbf; - nxt_buf_filter_t *f; - nxt_buf_filter_ctx_t *ctx; - - jbf = obj; - f = data; - b = jbf->buffer; - jbf->buffer = NULL; - - nxt_debug(task, "buf filter file completion: \"%FN\" %O-%O", - jbf->file.name, b->file_pos, b->file_end); - - f->run->job_file_release(f); - - ctx = nxt_mem_cache_alloc0(f->mem_pool, sizeof(nxt_buf_filter_ctx_t)); - if (nxt_slow_path(ctx == NULL)) { - goto fail; - } - - ctx->filter = f; - ctx->buf = f->input; - - f->input->file_pos = b->file_end; - - done = (f->input->file_pos == f->input->file_end); - - if (done) { - f->input = f->input->next; - f->reading = 0; - } - - b->data = f->data; - b->completion_handler = nxt_buf_filter_buf_completion; - b->parent = (nxt_buf_t *) ctx; - b->next = NULL; - - nxt_buf_chain_add(&f->current, b); - - nxt_buf_filter(task, f, NULL); - - if (b->mem.pos == b->mem.free) { - /* - * The buffer has been completely processed by nxt_buf_filter(), - * its completion handler has been placed in workqueue and - * nxt_buf_filter_buf_completion() should be eventually called. - */ - return; - } - - if (!done) { - /* Try to allocate another buffer and read the next file part. */ - nxt_buf_filter_file_read(task, f); - } - - return; - -fail: - - nxt_thread_work_queue_add(task->thread, f->work_queue, f->run->filter_error, - task, f, f->data); -} - - -static void -nxt_buf_filter_buf_completion(nxt_task_t *task, void *obj, void *data) -{ - nxt_buf_t *fb, *b; - nxt_buf_filter_t *f; - nxt_buf_filter_ctx_t *ctx; - - b = obj; - ctx = data; - f = ctx->filter; - - nxt_debug(task, "buf filter completion: %p \"%FN\" %O-%O", - b, f->filter_file->job_file.file.name, b->file_pos, b->file_end); - - /* nxt_http_send_filter() might clear a buffer's file status. */ - b->is_file = 1; - - fb = ctx->buf; - - nxt_mp_free(f->mem_pool, ctx); - nxt_buf_pool_free(&f->filter_file->buffers, b); - - if (fb->file_pos < fb->file_end) { - nxt_buf_filter_file_read(task, f); - return; - } - - if (b->file_end == fb->file_end) { - nxt_buf_pool_destroy(&f->filter_file->buffers); - - nxt_job_destroy(&f->filter_file->job_file.job); - - nxt_thread_work_queue_add(task->thread, f->work_queue, - fb->completion_handler, - task, fb, fb->parent); - } - - nxt_buf_filter(task, f, NULL); -} - - -static void -nxt_buf_filter_file_read_error(nxt_task_t *task, void *obj, void *data) -{ - nxt_buf_filter_t *f; - - f = data; - - nxt_thread_work_queue_add(task->thread, f->work_queue, f->run->filter_error, - task, f, f->data); -} diff --git a/src/nxt_buf_filter.h b/src/nxt_buf_filter.h deleted file mode 100644 index 27487baa..00000000 --- a/src/nxt_buf_filter.h +++ /dev/null @@ -1,115 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_BUF_FILTER_H_INCLUDED_ -#define _NXT_BUF_FILTER_H_INCLUDED_ - - -/* - * nxt_buf_filter is a framework intended to simplify processing file - * buffers content by a filter. The filter should set callbacks and - * call nxt_buf_filter_add() to start processing. - * - * At first buf_filter calls filter_ready() and the filter ensures - * it may allocate or reuse its internal buffer. No real allocation - * is performed at this step. - * - * TODO prevent unneeded allocaiton if no input data. - * - * - * TODO: The filter can flush data buffered - * previously, if all internal buffers are full. - * - * Then buf_filter looks buffer chains. There are two buffer chains: - * the input chain is a chain of original incoming memory, file, and sync - * buffers; and the current chain is a chain of memory/file buffers read - * from a file-only buffer. The current chain is processed first. Since - * buffers in this chain always contains a memory part, they can be passed - * one by one to the filter using filter_process(). If there is an output - * buffer after the buffer processing, it is added to output chain. The - * output buffers are not filter internal buffers. They just point to these - * internal buffers and one internal buffer can correspond to several output - * buffers which point to adjoining parts of the internal buffer. Further - * processing depends on filter_process() result code: if it returns NXT_OK, - * then the filter internal buffer is not full and buf_filter looks the next - * current or input buffer. If result code is NXT_AGAIN, then the filter - * internal buffer is full and buf_filter calls filter_flush() and then - * schedules to run nxt_buf_filter_repeat(). nxt_buf_filter_repeat() will - * run after all ready output buffer completion handlers and will call - * buf_filter again if no one completion handler will do it already using - * nxt_buf_filter_enqueue(). So in any case buf_filter will run again only - * once. - * - * TODO: - * in ideal just one the filter internal buffer. - * This allows to minimize number of the filter internal buffers if they - * flush fast. - * - * If the current chain is empty, the buf_filter processes the input chain. - * Memory buffers are passed to the filter using filter_process(). If an - * input buffer is a file buffer, then buf_filter calls filter_flush() - * and starts a file job to read the buffer in memory. The file job reads - * file parts into memory/file buffers and adds them to the current chain. - * - * Sync buffers are passed to the filter using filter_sync(). Its - * post-processing is similar to the filter_process() post-processing, - * except sync buffers are always added unmodified to the output chain. - */ - -typedef struct { - nxt_job_file_t job_file; - nxt_buf_pool_t buffers; -} nxt_buf_filter_file_t; - - -typedef struct nxt_buf_filter_s nxt_buf_filter_t; - -typedef struct { - nxt_int_t (*filter_ready)(nxt_buf_filter_t *f); - nxt_int_t (*filter_process)(nxt_buf_filter_t *f); - nxt_int_t (*filter_flush)(nxt_buf_filter_t *f); - - nxt_int_t (*filter_sync_nobuf)(nxt_buf_filter_t *f); - nxt_int_t (*filter_sync_flush)(nxt_buf_filter_t *f); - nxt_int_t (*filter_sync_last)(nxt_buf_filter_t *f); - - void (*filter_next)(nxt_buf_filter_t *f); - nxt_work_handler_t filter_error; - - nxt_buf_filter_file_t *(*job_file_create)(nxt_buf_filter_t *f); - void (*job_file_retain)(nxt_buf_filter_t *f); - void (*job_file_release)(nxt_buf_filter_t *f); -} nxt_buf_filter_ops_t; - - -struct nxt_buf_filter_s { - nxt_buf_t *current; - nxt_buf_t *input; - nxt_buf_t *output; - nxt_buf_t *last; - - nxt_work_queue_t *work_queue; - nxt_buf_filter_file_t *filter_file; - void *data; - nxt_mp_t *mem_pool; - - const nxt_buf_filter_ops_t *run; - - uint8_t mmap; /* 1 bit */ - uint8_t done; /* 1 bit */ - uint8_t queued; /* 1 bit */ - uint8_t reading; /* 1 bit */ - uint8_t buffering; /* 1 bit */ -}; - - -NXT_EXPORT void nxt_buf_filter_add(nxt_task_t *task, nxt_buf_filter_t *f, - nxt_buf_t *b); -NXT_EXPORT void nxt_buf_filter(nxt_task_t *task, void *obj, void *data); -NXT_EXPORT void nxt_buf_filter_enqueue(nxt_task_t *task, nxt_buf_filter_t *f); - - -#endif /* _NXT_BUF_FILTER_H_INCLUDED_ */ diff --git a/src/nxt_cache.c b/src/nxt_cache.c deleted file mode 100644 index e81d63dc..00000000 --- a/src/nxt_cache.c +++ /dev/null @@ -1,642 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -/* A cache time resolution is 10ms. */ -#define nxt_cache_time(thr) \ - (uint64_t) (nxt_thread_time(thr) * 100) - - -static nxt_int_t nxt_cache_lvlhsh_test(nxt_lvlhsh_query_t *lhq, void *data); -static nxt_work_handler_t nxt_cache_query_locked(nxt_cache_t *cache, - nxt_cache_query_t *q, nxt_lvlhsh_query_t *lhq); -static nxt_work_handler_t nxt_cache_node_hold(nxt_cache_t *cache, - nxt_cache_query_t *q, nxt_lvlhsh_query_t *lhq); -static nxt_work_handler_t nxt_cache_node_test(nxt_cache_t *cache, - nxt_cache_query_t *q); - -static void nxt_cache_wait_handler(nxt_thread_t *thr, void *obj, void *data); -static void nxt_cache_timeout_handler(nxt_thread_t *thr, void *obj, void *data); -static void nxt_cache_wake_handler(nxt_thread_t *thr, void *obj, void *data); -static ssize_t nxt_cache_release_locked(nxt_cache_t *cache, - nxt_cache_query_t *q, u_char *buf, size_t size); - -static nxt_cache_node_t *nxt_cache_node_alloc(nxt_cache_t *cache); -static void nxt_cache_node_free(nxt_cache_t *cache, nxt_cache_node_t *node, - nxt_bool_t fast); -static nxt_cache_query_wait_t *nxt_cache_query_wait_alloc(nxt_cache_t *cache, - nxt_bool_t *slow); -static void nxt_cache_query_wait_free(nxt_cache_t *cache, - nxt_cache_query_wait_t *qw); - - -/* STUB */ -nxt_int_t nxt_cache_shm_create(nxt_mem_zone_t *pool); -static void *nxt_cache_shm_alloc(void *data, size_t size, nxt_uint_t nalloc); -/**/ - - -nxt_int_t -nxt_cache_shm_create(nxt_mem_zone_t *mz) -{ - nxt_cache_t *cache; - - static const nxt_lvlhsh_proto_t proto nxt_aligned(64) = { - NXT_LVLHSH_LARGE_SLAB, - 0, - nxt_cache_lvlhsh_test, - (nxt_lvlhsh_alloc_t) nxt_cache_shm_alloc, - (nxt_lvlhsh_free_t) nxt_mem_zone_free, - }; - - cache = nxt_mem_zone_zalloc(mz, sizeof(nxt_cache_t)); - - if (cache == NULL) { - return NXT_ERROR; - } - - cache->proto = &proto; - cache->pool = mz; - - cache->start_time = nxt_cache_time(nxt_thread()); - - return NXT_OK; -} - - -static void * -nxt_cache_shm_alloc(void *data, size_t size, nxt_uint_t nalloc) -{ - return nxt_mem_zone_align(data, size, size); -} - - -void -nxt_cache_init(nxt_cache_t *cache) -{ - static const nxt_lvlhsh_proto_t proto nxt_aligned(64) = { - NXT_LVLHSH_LARGE_MEMALIGN, - 0, - nxt_cache_lvlhsh_test, - nxt_lvlhsh_alloc, - nxt_lvlhsh_free, - }; - - cache->proto = &proto; - - cache->start_time = nxt_cache_time(nxt_thread()); -} - - -static nxt_int_t -nxt_cache_lvlhsh_test(nxt_lvlhsh_query_t *lhq, void *data) -{ - nxt_cache_node_t *node; - - node = data; - - if (nxt_str_eq(&lhq->key, node->key_data, node->key_len)) { - return NXT_OK; - } - - return NXT_DECLINED; -} - - -nxt_inline void -nxt_cache_lock(nxt_cache_t *cache) -{ - if (cache->shared) { - nxt_thread_spin_lock(&cache->lock); - } -} - - -nxt_inline void -nxt_cache_unlock(nxt_cache_t *cache) -{ - if (cache->shared) { - nxt_thread_spin_unlock(&cache->lock); - } -} - - -void -nxt_cache_query(nxt_cache_t *cache, nxt_cache_query_t *q) -{ - nxt_thread_t *thr; - nxt_lvlhsh_query_t lhq; - nxt_work_handler_t handler; - - thr = nxt_thread(); - - if (cache != NULL) { - lhq.key_hash = nxt_murmur_hash2(q->key_data, q->key_len); - lhq.replace = 0; - lhq.key.len = q->key_len; - lhq.key.data = q->key_data; - lhq.proto = cache->proto; - lhq.pool = cache->pool; - - q->now = nxt_cache_time(thr); - - nxt_cache_lock(cache); - - handler = nxt_cache_query_locked(cache, q, &lhq); - - nxt_cache_unlock(cache); - - } else { - handler = q->state->nocache_handler; - } - - handler(thr, q, NULL); -} - - -static nxt_work_handler_t -nxt_cache_query_locked(nxt_cache_t *cache, nxt_cache_query_t *q, - nxt_lvlhsh_query_t *lhq) -{ - nxt_int_t ret; - nxt_time_t expiry; - nxt_cache_node_t *node; - nxt_cache_query_state_t *state; - - if (q->hold) { - return nxt_cache_node_hold(cache, q, lhq); - } - - ret = nxt_lvlhsh_find(&cache->lvlhsh, lhq); - - state = q->state; - - if (ret != NXT_OK) { - /* NXT_DECLINED */ - return state->nocache_handler; - } - - node = lhq->value; - node->count++; - q->node = node; - - expiry = cache->start_time + node->expiry; - - if (q->now < expiry) { - return state->ready_handler; - } - - q->stale = 1; - - return state->stale_handler; -} - - -static nxt_work_handler_t -nxt_cache_node_hold(nxt_cache_t *cache, nxt_cache_query_t *q, - nxt_lvlhsh_query_t *lhq) -{ - nxt_int_t ret; - nxt_bool_t slow; - nxt_cache_node_t *node, *sentinel; - nxt_work_handler_t handler; - nxt_cache_query_wait_t *qw; - nxt_cache_query_state_t *state; - - state = q->state; - sentinel = nxt_cache_node_alloc(cache); - - if (nxt_slow_path(sentinel == NULL)) { - return state->error_handler; - } - - sentinel->key_data = q->key_data; - sentinel->key_len = q->key_len; - lhq->value = sentinel; - - /* - * Try to insert an empty sentinel node to hold updating - * process if there is no existent cache node in cache. - */ - ret = nxt_lvlhsh_insert(&cache->lvlhsh, lhq); - - if (ret == NXT_OK) { - /* The sentinel node was successully added. */ - - q->node = sentinel; - sentinel->updating = 1; - return state->update_handler; - } - - nxt_cache_node_free(cache, sentinel, 1); - - if (ret == NXT_ERROR) { - return state->error_handler; - } - - /* NXT_DECLINED: a cache node exists. */ - - node = lhq->value; - node->count++; - q->node = node; - - handler = nxt_cache_node_test(cache, q); - if (handler != NULL) { - return handler; - } - - /* Add the node to a wait queue. */ - - qw = nxt_cache_query_wait_alloc(cache, &slow); - if (nxt_slow_path(qw == NULL)) { - return state->error_handler; - } - - if (slow) { - /* The node state may have been changed during slow allocation. */ - - handler = nxt_cache_node_test(cache, q); - if (handler != NULL) { - nxt_cache_query_wait_free(cache, qw); - return handler; - } - } - - qw->query = q; - qw->next = node->waiting; - qw->busy = 0; - qw->deleted = 0; - qw->pid = nxt_pid; - qw->engine = nxt_thread_event_engine(); - qw->handler = nxt_cache_wake_handler; - qw->cache = cache; - - node->waiting = qw; - - return nxt_cache_wait_handler; -} - - -static nxt_work_handler_t -nxt_cache_node_test(nxt_cache_t *cache, nxt_cache_query_t *q) -{ - nxt_time_t expiry; - nxt_cache_node_t *node; - nxt_cache_query_state_t *state; - - q->stale = 0; - state = q->state; - node = q->node; - - expiry = cache->start_time + node->expiry; - - if (q->now < expiry) { - return state->ready_handler; - } - - /* - * A valid stale or empty sentinel cache node. - * The sentinel node can be only in updating state. - */ - - if (node->updating) { - - if (node->expiry != 0) { - /* A valid stale cache node. */ - - q->stale = 1; - - if (q->use_stale) { - return state->stale_handler; - } - } - - /* A sentinel node. */ - return NULL; - } - - /* A valid stale cache node is not being updated now. */ - - q->stale = 1; - - if (q->use_stale) { - - if (q->update_stale) { - node->updating = 1; - return state->update_stale_handler; - } - - return state->stale_handler; - } - - node->updating = 1; - return state->update_handler; -} - - -static void -nxt_cache_wait_handler(nxt_thread_t *thr, void *obj, void *data) -{ - nxt_event_timer_t *ev; - nxt_cache_query_t *cq; - - cq = obj; - - if (cq->timeout != 0) { - - ev = &cq->timer; - - if (ev->state == NXT_EVENT_TIMER_DISABLED) { - ev->handler = nxt_cache_timeout_handler; - nxt_event_timer_ident(ev, -1); - - nxt_event_timer_add(thr->engine, ev, cq->timeout); - } - } -} - - -static void -nxt_cache_timeout_handler(nxt_thread_t *thr, void *obj, void *data) -{ - nxt_cache_query_t *cq; - nxt_event_timer_t *ev; - - ev = obj; - - cq = nxt_event_timer_data(ev, nxt_cache_query_t, timer); - - cq->state->timeout_handler(thr, cq, NULL); -} - - -static void -nxt_cache_wake_handler(nxt_thread_t *thr, void *obj, void *data) -{ - nxt_cache_t *cache; - nxt_work_handler_t handler; - nxt_cache_query_t *q; - nxt_cache_query_wait_t *qw; - - qw = obj; - q = qw->query; - cache = qw->cache; - - nxt_cache_lock(cache); - - handler = nxt_cache_node_test(cache, q); - - if (handler != NULL) { - nxt_cache_query_wait_free(cache, qw); - - } else { - /* Wait again. */ - qw->next = q->node->waiting; - q->node->waiting = qw; - } - - nxt_cache_unlock(cache); - - handler(thr, q, NULL); -} - - -nxt_int_t -nxt_cache_update(nxt_cache_t *cache, nxt_cache_query_t *q) -{ - nxt_int_t ret; - nxt_cache_node_t *node; - nxt_lvlhsh_query_t lhq; - - node = q->node; - - node->accessed = nxt_cache_time(nxt_thread()) - cache->start_time; - - node->updating = 0; - node->count = 1; - - lhq.key_hash = nxt_murmur_hash2(node->key_data, node->key_len); - lhq.replace = 1; - lhq.key.len = node->key_len; - lhq.key.data = node->key_data; - lhq.value = node; - lhq.proto = cache->proto; - lhq.pool = cache->pool; - - nxt_cache_lock(cache); - - ret = nxt_lvlhsh_insert(&cache->lvlhsh, &lhq); - - if (nxt_fast_path(ret != NXT_OK)) { - - nxt_queue_insert_head(&cache->expiry_queue, &node->link); - - node = lhq.value; - - if (node != NULL) { - /* A replaced node. */ - - nxt_queue_remove(&node->link); - - if (node->count != 0) { - node->deleted = 1; - - } else { - // delete cache node - } - } - } - - nxt_cache_unlock(cache); - - return ret; -} - - -void -nxt_cache_release(nxt_cache_t *cache, nxt_cache_query_t *q) -{ - u_char *p, *data; - size_t size; - ssize_t ret; - nxt_thread_t *thr; - u_char buf[1024]; - - thr = nxt_thread(); - q->now = nxt_cache_time(thr); - - p = buf; - size = sizeof(buf); - - for ( ;; ) { - nxt_cache_lock(cache); - - ret = nxt_cache_release_locked(cache, q, p, size); - - nxt_cache_unlock(cache); - - if (ret == 0) { - return; - } - - size = nxt_abs(ret); - - data = nxt_malloc(size); - - if (data == NULL) { - /* TODO: retry */ - return; - } - - if (ret < 0) { - p = data; - continue; - } - - if (p != data) { - nxt_memcpy(data, p, size); - } - - nxt_thread_work_queue_add(thr, &thr->work_queue.main, - cache->delete_handler, data, NULL, thr->log); - } -} - - -static ssize_t -nxt_cache_release_locked(nxt_cache_t *cache, nxt_cache_query_t *q, - u_char *buf, size_t size) -{ - ssize_t ret; - nxt_cache_node_t *node; - - node = q->node; - node->count--; - - if (node->count != 0) { - return 0; - } - - if (!node->deleted) { - /* - * A cache node is locked whilst its count is non zero. - * To minimize number of operations the node's place in expiry - * queue can be updated only if the node is not currently used. - */ - node->accessed = q->now - cache->start_time; - - nxt_queue_remove(&node->link); - nxt_queue_insert_head(&cache->expiry_queue, &node->link); - - return 0; - } - - ret = 0; -#if 0 - - ret = cache->delete_copy(cache, node, buf, size); - - if (ret < 0) { - return ret; - } - -#endif - - nxt_cache_node_free(cache, node, 0); - - return ret; -} - - -static nxt_cache_node_t * -nxt_cache_node_alloc(nxt_cache_t *cache) -{ - nxt_queue_link_t *link; - nxt_cache_node_t *node; - - link = nxt_queue_first(&cache->free_nodes); - - if (nxt_fast_path(link != nxt_queue_tail(&cache->free_nodes))) { - cache->nfree_nodes--; - nxt_queue_remove(link); - - node = nxt_queue_link_data(link, nxt_cache_node_t, link); - nxt_memzero(node, sizeof(nxt_cache_node_t)); - - return node; - } - - nxt_cache_unlock(cache); - - node = cache->alloc(cache->data, sizeof(nxt_cache_node_t)); - - nxt_cache_lock(cache); - - return node; -} - - -static void -nxt_cache_node_free(nxt_cache_t *cache, nxt_cache_node_t *node, nxt_bool_t fast) -{ - if (fast || cache->nfree_nodes < 32) { - nxt_queue_insert_head(&cache->free_nodes, &node->link); - cache->nfree_nodes++; - return; - } - - nxt_cache_unlock(cache); - - cache->free(cache->data, node); - - nxt_cache_lock(cache); -} - - -static nxt_cache_query_wait_t * -nxt_cache_query_wait_alloc(nxt_cache_t *cache, nxt_bool_t *slow) -{ - nxt_cache_query_wait_t *qw; - - qw = cache->free_query_wait; - - if (nxt_fast_path(qw != NULL)) { - cache->free_query_wait = qw->next; - cache->nfree_query_wait--; - - *slow = 0; - return qw; - } - - nxt_cache_unlock(cache); - - qw = cache->alloc(cache->data, sizeof(nxt_cache_query_wait_t)); - *slow = 1; - - nxt_cache_lock(cache); - - return qw; -} - - -static void -nxt_cache_query_wait_free(nxt_cache_t *cache, nxt_cache_query_wait_t *qw) -{ - if (cache->nfree_query_wait < 32) { - qw->next = cache->free_query_wait; - cache->free_query_wait = qw; - cache->nfree_query_wait++; - return; - } - - nxt_cache_unlock(cache); - - cache->free(cache->data, qw); - - nxt_cache_lock(cache); -} diff --git a/src/nxt_cache.h b/src/nxt_cache.h deleted file mode 100644 index 567b5581..00000000 --- a/src/nxt_cache.h +++ /dev/null @@ -1,122 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_CACHE_INCLUDED_ -#define _NXT_CACHE_INCLUDED_ - - -typedef struct nxt_cache_query_s nxt_cache_query_t; -typedef struct nxt_cache_query_wait_s nxt_cache_query_wait_t; - - -typedef struct { - uint32_t shared; /* 1 bit */ - nxt_thread_spinlock_t lock; - - nxt_lvlhsh_t lvlhsh; - const nxt_lvlhsh_proto_t *proto; - void *pool; - - nxt_queue_t expiry_queue; - - nxt_queue_t free_nodes; - uint32_t nfree_nodes; - - uint32_t nfree_query_wait; - nxt_cache_query_wait_t *free_query_wait; - - uint64_t start_time; - - /* STUB: use nxt_lvlhsh_proto_t */ - void *(*alloc)(void *data, size_t size); - void (*free)(void *data, void *p); - void *data; - - nxt_work_handler_t delete_handler; -} nxt_cache_t; - - -typedef struct { - u_char *key_data; - - uint16_t key_len; /* 16 bits */ - uint8_t uses; /* 8 bits */ - uint8_t updating:1; - uint8_t deleted:1; - - uint32_t count; - - /* Times relative to the cache->start_time. */ - uint32_t expiry; - uint32_t accessed; - - nxt_off_t size; - - nxt_queue_link_t link; - - nxt_cache_query_wait_t *waiting; -} nxt_cache_node_t; - - -struct nxt_cache_query_wait_s { - nxt_cache_query_t *query; - nxt_cache_query_wait_t *next; - - uint8_t busy; /* 1 bit */ - uint8_t deleted; /* 1 bit */ - - nxt_pid_t pid; - nxt_event_engine_t *engine; - nxt_work_handler_t handler; - nxt_cache_t *cache; -}; - - -typedef struct { - nxt_work_handler_t nocache_handler; - nxt_work_handler_t ready_handler; - nxt_work_handler_t stale_handler; - nxt_work_handler_t update_stale_handler; - nxt_work_handler_t update_handler; - nxt_work_handler_t timeout_handler; - nxt_work_handler_t error_handler; -} nxt_cache_query_state_t; - - -struct nxt_cache_query_s { - u_char *key_data; - - uint16_t key_len; /* 16 bits */ -#if (NXT_64_BIT) - uint8_t hold; /* 1 bit */ - uint8_t use_stale; /* 1 bit */ - uint8_t update_stale; /* 1 bit */ - uint8_t stale; /* 1 bit */ -#else - uint8_t hold:1; - uint8_t use_stale:1; - uint8_t update_stale:1; - uint8_t stale:1; -#endif - - nxt_cache_node_t *node; - nxt_cache_query_t *next; - nxt_cache_query_state_t *state; - - nxt_time_t now; - - nxt_msec_t timeout; - nxt_timer_t timer; -}; - - -NXT_EXPORT void nxt_cache_init(nxt_cache_t *cache); -NXT_EXPORT void nxt_cache_query(nxt_cache_t *cache, nxt_cache_query_t *q); -NXT_EXPORT void nxt_cache_release(nxt_cache_t *cache, nxt_cache_query_t *q); -NXT_EXPORT nxt_int_t nxt_cache_update(nxt_cache_t *cache, nxt_cache_query_t *q); - - -#endif /* _NXT_CACHE_INCLUDED_ */ diff --git a/src/nxt_fastcgi_record_parse.c b/src/nxt_fastcgi_record_parse.c deleted file mode 100644 index 7d2ce32e..00000000 --- a/src/nxt_fastcgi_record_parse.c +++ /dev/null @@ -1,307 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -#define NXT_FASTCGI_DATA_MIDDLE 0 -#define NXT_FASTCGI_DATA_END_ON_BORDER 1 -#define NXT_FASTCGI_DATA_END 2 - - -static nxt_int_t nxt_fastcgi_buffer(nxt_fastcgi_parse_t *fp, nxt_buf_t ***tail, - nxt_buf_t *in); - - -void -nxt_fastcgi_record_parse(nxt_task_t *task, nxt_fastcgi_parse_t *fp, - nxt_buf_t *in) -{ - u_char ch; - nxt_int_t ret, stream; - nxt_buf_t *b, *nb, **tail[2]; - const char *msg; - enum { - sw_fastcgi_version = 0, - sw_fastcgi_type, - sw_fastcgi_request_id_high, - sw_fastcgi_request_id_low, - sw_fastcgi_content_length_high, - sw_fastcgi_content_length_low, - sw_fastcgi_padding_length, - sw_fastcgi_reserved, - sw_fastcgi_data, - sw_fastcgi_padding, - sw_fastcgi_end_request, - } state; - - fp->out[0] = NULL; - fp->out[1] = NULL; - - tail[0] = &fp->out[0]; - tail[1] = &fp->out[1]; - - state = fp->state; - - for (b = in; b != NULL; b = b->next) { - - if (nxt_buf_is_sync(b)) { - **tail = b; - *tail = &b->next; - continue; - } - - fp->pos = b->mem.pos; - - while (fp->pos < b->mem.free) { - /* - * The sw_fastcgi_data state is tested outside the - * switch to preserve fp->pos and to not touch memory. - */ - if (state == sw_fastcgi_data) { - - /* - * fp->type here can be only NXT_FASTCGI_STDOUT - * or NXT_FASTCGI_STDERR. NXT_FASTCGI_END_REQUEST - * is tested in sw_fastcgi_reserved. - */ - stream = fp->type - NXT_FASTCGI_STDOUT; - - ret = nxt_fastcgi_buffer(fp, &tail[stream], b); - - if (ret == NXT_FASTCGI_DATA_MIDDLE) { - goto next; - } - - if (nxt_slow_path(ret == NXT_ERROR)) { - fp->error = 1; - goto done; - } - - if (fp->padding == 0) { - state = sw_fastcgi_version; - - } else { - state = sw_fastcgi_padding; - } - - if (ret == NXT_FASTCGI_DATA_END_ON_BORDER) { - goto next; - } - - /* ret == NXT_FASTCGI_DATA_END */ - } - - ch = *fp->pos++; - - nxt_thread_log_debug("fastcgi record byte: %02Xd", ch); - - switch (state) { - - case sw_fastcgi_version: - if (nxt_fast_path(ch == 1)) { - state = sw_fastcgi_type; - continue; - } - - msg = "unsupported FastCGI protocol version"; - goto fastcgi_error; - - case sw_fastcgi_type: - switch (ch) { - case NXT_FASTCGI_STDOUT: - case NXT_FASTCGI_STDERR: - case NXT_FASTCGI_END_REQUEST: - fp->type = ch; - state = sw_fastcgi_request_id_high; - continue; - default: - msg = "invalid FastCGI record type"; - goto fastcgi_error; - } - - case sw_fastcgi_request_id_high: - /* FastCGI multiplexing is not supported. */ - if (nxt_fast_path(ch == 0)) { - state = sw_fastcgi_request_id_low; - continue; - } - - msg = "unexpected FastCGI request ID high byte"; - goto fastcgi_error; - - case sw_fastcgi_request_id_low: - if (nxt_fast_path(ch == 1)) { - state = sw_fastcgi_content_length_high; - continue; - } - - msg = "unexpected FastCGI request ID low byte"; - goto fastcgi_error; - - case sw_fastcgi_content_length_high: - fp->length = ch << 8; - state = sw_fastcgi_content_length_low; - continue; - - case sw_fastcgi_content_length_low: - fp->length |= ch; - state = sw_fastcgi_padding_length; - continue; - - case sw_fastcgi_padding_length: - fp->padding = ch; - state = sw_fastcgi_reserved; - continue; - - case sw_fastcgi_reserved: - nxt_thread_log_debug("fastcgi record type:%d " - "length:%uz padding:%d", - fp->type, fp->length, fp->padding); - - if (nxt_fast_path(fp->type != NXT_FASTCGI_END_REQUEST)) { - state = sw_fastcgi_data; - continue; - } - - state = sw_fastcgi_end_request; - continue; - - case sw_fastcgi_data: - /* - * This state is processed before the switch. - * It added here just to suppress a warning. - */ - continue; - - case sw_fastcgi_padding: - /* - * No special fast processing of padding - * because it usually takes just 1-7 bytes. - */ - fp->padding--; - - if (fp->padding == 0) { - nxt_thread_log_debug("fastcgi record end"); - state = sw_fastcgi_version; - } - continue; - - case sw_fastcgi_end_request: - /* Just skip 8 bytes of END_REQUEST. */ - fp->length--; - - if (fp->length != 0) { - continue; - } - - fp->done = 1; - - nxt_thread_log_debug("fastcgi end request"); - - goto done; - } - } - - if (b->retain == 0) { - /* No record data was found in a buffer. */ - nxt_thread_current_work_queue_add(task->thread, - b->completion_handler, - task, b, b->parent); - } - - next: - - continue; - } - - fp->state = state; - - return; - -fastcgi_error: - - nxt_thread_log_error(NXT_LOG_ERR, "upstream sent %s: %d", msg, ch); - - fp->fastcgi_error = 1; - -done: - - nb = fp->last_buf(fp); - - if (nxt_fast_path(nb != NULL)) { - *tail[0] = nb; - - } else { - fp->error = 1; - } - - // STUB: fp->fastcgi_error = 1; - // STUB: fp->error = 1; - - return; -} - - -static nxt_int_t -nxt_fastcgi_buffer(nxt_fastcgi_parse_t *fp, nxt_buf_t ***tail, nxt_buf_t *in) -{ - u_char *p; - size_t size; - nxt_buf_t *b; - - if (fp->length == 0) { - return NXT_FASTCGI_DATA_END; - } - - p = fp->pos; - size = in->mem.free - p; - - if (fp->length >= size && in->retain == 0) { - /* - * Use original buffer if the buffer is lesser than or equal to - * FastCGI record size and this is the first record in the buffer. - */ - in->mem.pos = p; - **tail = in; - *tail = &in->next; - - } else { - b = nxt_buf_mem_alloc(fp->mem_pool, 0, 0); - if (nxt_slow_path(b == NULL)) { - return NXT_ERROR; - } - - **tail = b; - *tail = &b->next; - - b->parent = in; - in->retain++; - b->mem.pos = p; - b->mem.start = p; - - if (fp->length < size) { - p += fp->length; - fp->pos = p; - - b->mem.free = p; - b->mem.end = p; - - return NXT_FASTCGI_DATA_END; - } - - b->mem.free = in->mem.free; - b->mem.end = in->mem.free; - } - - fp->length -= size; - - if (fp->length == 0) { - return NXT_FASTCGI_DATA_END_ON_BORDER; - } - - return NXT_FASTCGI_DATA_MIDDLE; -} diff --git a/src/nxt_fastcgi_source.c b/src/nxt_fastcgi_source.c deleted file mode 100644 index b2424292..00000000 --- a/src/nxt_fastcgi_source.c +++ /dev/null @@ -1,750 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -#define NXT_FASTCGI_RESPONDER 1 -#define NXT_FASTCGI_KEEP_CONN 1 - - -typedef struct { - u_char *buf; - uint32_t len; - u_char length[4]; -} nxt_fastcgi_param_t; - - -#define nxt_fastcgi_set_record_length(p, length) \ - do { \ - uint32_t len = length; \ - \ - p[1] = (u_char) len; len >>= 8; \ - p[0] = (u_char) len; \ - } while (0) - - -nxt_inline size_t -nxt_fastcgi_param_length(u_char *p, uint32_t length) -{ - if (nxt_fast_path(length < 128)) { - *p = (u_char) length; - return 1; - } - - p[3] = (u_char) length; length >>= 8; - p[2] = (u_char) length; length >>= 8; - p[1] = (u_char) length; length >>= 8; - p[0] = (u_char) (length | 0x80); - - return 4; -} - - -static nxt_buf_t *nxt_fastcgi_request_create(nxt_fastcgi_source_t *fs); -static nxt_int_t nxt_fastcgi_next_param(nxt_fastcgi_source_t *fs, - nxt_fastcgi_param_t *param); - -static void nxt_fastcgi_source_record_filter(nxt_task_t *task, void *obj, - void *data); -static void nxt_fastcgi_source_record_error(nxt_task_t *task, void *obj, - void *data); -static void nxt_fastcgi_source_header_filter(nxt_task_t *task, void *obj, - void *data); -static void nxt_fastcgi_source_sync_buffer(nxt_task_t *task, - nxt_fastcgi_source_t *fs, nxt_buf_t *b); - -static nxt_int_t nxt_fastcgi_source_header_process(nxt_task_t *task, - nxt_fastcgi_source_t *fs); -static nxt_int_t nxt_fastcgi_source_status(nxt_upstream_source_t *us, - nxt_name_value_t *nv); -static nxt_int_t nxt_fastcgi_source_content_length(nxt_upstream_source_t *us, - nxt_name_value_t *nv); - -static void nxt_fastcgi_source_header_ready(nxt_fastcgi_source_t *fs, - nxt_buf_t *b); -static void nxt_fastcgi_source_body_filter(nxt_task_t *task, void *obj, - void *data); -static nxt_buf_t *nxt_fastcgi_source_last_buf(nxt_fastcgi_parse_t *fp); -static void nxt_fastcgi_source_error(nxt_task_t *task, - nxt_stream_source_t *stream); -static void nxt_fastcgi_source_fail(nxt_task_t *task, nxt_fastcgi_source_t *fs); - - -/* - * A FastCGI request: - * FCGI_BEGIN_REQUEST record; - * Several FCGI_PARAMS records, the last FCGI_PARAMS record must have - * zero content length, - * Several FCGI_STDIN records, the last FCGI_STDIN record must have - * zero content length. - */ - -static const uint8_t nxt_fastcgi_begin_request[] = { - 1, /* FastCGI version. */ - NXT_FASTCGI_BEGIN_REQUEST, /* The BEGIN_REQUEST record type. */ - 0, 1, /* Request ID. */ - 0, 8, /* Content length of the Role record. */ - 0, /* Padding length. */ - 0, /* Reserved. */ - - 0, NXT_FASTCGI_RESPONDER, /* The Responder Role. */ - 0, /* Flags. */ - 0, 0, 0, 0, 0, /* Reserved. */ -}; - - -static const uint8_t nxt_fastcgi_params_record[] = { - 1, /* FastCGI version. */ - NXT_FASTCGI_PARAMS, /* The PARAMS record type. */ - 0, 1, /* Request ID. */ - 0, 0, /* Content length. */ - 0, /* Padding length. */ - 0, /* Reserved. */ -}; - - -static const uint8_t nxt_fastcgi_stdin_record[] = { - 1, /* FastCGI version. */ - NXT_FASTCGI_STDIN, /* The STDIN record type. */ - 0, 1, /* Request ID. */ - 0, 0, /* Content length. */ - 0, /* Padding length. */ - 0, /* Reserved. */ -}; - - -void -nxt_fastcgi_source_handler(nxt_task_t *task, nxt_upstream_source_t *us, - nxt_fastcgi_source_request_create_t request_create) -{ - nxt_stream_source_t *stream; - nxt_fastcgi_source_t *fs; - - fs = nxt_mp_zget(us->buffers.mem_pool, sizeof(nxt_fastcgi_source_t)); - if (nxt_slow_path(fs == NULL)) { - goto fail; - } - - us->protocol_source = fs; - - fs->header_in.list = nxt_list_create(us->buffers.mem_pool, 8, - sizeof(nxt_name_value_t)); - if (nxt_slow_path(fs->header_in.list == NULL)) { - goto fail; - } - - fs->header_in.hash = us->header_hash; - fs->upstream = us; - fs->request_create = request_create; - - stream = us->stream; - - if (stream == NULL) { - stream = nxt_mp_zget(us->buffers.mem_pool, sizeof(nxt_stream_source_t)); - if (nxt_slow_path(stream == NULL)) { - goto fail; - } - - us->stream = stream; - stream->upstream = us; - - } else { - nxt_memzero(stream, sizeof(nxt_stream_source_t)); - } - - /* - * Create the FastCGI source filter chain: - * stream source | FastCGI record filter | FastCGI HTTP header filter - */ - stream->next = &fs->query; - stream->error_handler = nxt_fastcgi_source_error; - - fs->record.next.context = fs; - fs->record.next.filter = nxt_fastcgi_source_header_filter; - - fs->record.parse.last_buf = nxt_fastcgi_source_last_buf; - fs->record.parse.data = fs; - fs->record.parse.mem_pool = us->buffers.mem_pool; - - fs->query.context = &fs->record.parse; - fs->query.filter = nxt_fastcgi_source_record_filter; - - fs->header_in.content_length = -1; - - stream->out = nxt_fastcgi_request_create(fs); - - if (nxt_fast_path(stream->out != NULL)) { - nxt_memzero(&fs->u.header, sizeof(nxt_http_split_header_parse_t)); - fs->u.header.mem_pool = fs->upstream->buffers.mem_pool; - - nxt_stream_source_connect(task, stream); - return; - } - -fail: - - nxt_fastcgi_source_fail(task, fs); -} - - -static nxt_buf_t * -nxt_fastcgi_request_create(nxt_fastcgi_source_t *fs) -{ - u_char *p, *record_length; - size_t len, size, max_record_size; - nxt_int_t ret; - nxt_buf_t *b, *req, **prev; - nxt_bool_t begin_request; - nxt_fastcgi_param_t param; - - nxt_thread_log_debug("fastcgi request"); - - begin_request = 1; - param.len = 0; - prev = &req; - -new_buffer: - - ret = nxt_buf_pool_mem_alloc(&fs->upstream->buffers, 0); - if (nxt_slow_path(ret != NXT_OK)) { - return NULL; - } - - b = fs->upstream->buffers.current; - fs->upstream->buffers.current = NULL; - - *prev = b; - prev = &b->next; - -new_record: - - size = b->mem.end - b->mem.free; - size = nxt_align_size(size, 8) - 8; - /* The maximal FastCGI record content size is 65535. 65528 is 64K - 8. */ - max_record_size = nxt_min(65528, size); - - p = b->mem.free; - - if (begin_request) { - /* TODO: fastcgi keep conn in flags. */ - p = nxt_cpymem(p, nxt_fastcgi_begin_request, 16); - max_record_size -= 16; - begin_request = 0; - } - - b->mem.free = nxt_cpymem(p, nxt_fastcgi_params_record, 8); - record_length = &p[4]; - size = 0; - - for ( ;; ) { - if (param.len == 0) { - ret = nxt_fastcgi_next_param(fs, ¶m); - - if (nxt_slow_path(ret != NXT_OK)) { - - if (nxt_slow_path(ret == NXT_ERROR)) { - return NULL; - } - - /* ret == NXT_DONE */ - break; - } - } - - len = max_record_size; - - if (nxt_fast_path(len >= param.len)) { - len = param.len; - param.len = 0; - - } else { - param.len -= len; - } - - nxt_thread_log_debug("fastcgi copy len:%uz", len); - - b->mem.free = nxt_cpymem(b->mem.free, param.buf, len); - - size += len; - max_record_size -= len; - - if (nxt_slow_path(param.len != 0)) { - /* The record is full. */ - - param.buf += len; - - nxt_thread_log_debug("fastcgi content size:%uz", size); - - nxt_fastcgi_set_record_length(record_length, size); - - /* The minimal size of aligned record with content is 16 bytes. */ - if (b->mem.end - b->mem.free >= 16) { - goto new_record; - } - - nxt_thread_log_debug("\"%*s\"", b->mem.free - b->mem.pos, - b->mem.pos); - goto new_buffer; - } - } - - nxt_thread_log_debug("fastcgi content size:%uz", size); - - nxt_fastcgi_set_record_length(record_length, size); - - /* A padding length. */ - size = 8 - size % 8; - record_length[2] = (u_char) size; - nxt_memzero(b->mem.free, size); - b->mem.free += size; - - nxt_thread_log_debug("fastcgi padding:%uz", size); - - if (b->mem.end - b->mem.free < 16) { - nxt_thread_log_debug("\"%*s\"", b->mem.free - b->mem.pos, b->mem.pos); - - b = nxt_buf_mem_alloc(fs->upstream->buffers.mem_pool, 16, 0); - if (nxt_slow_path(b == NULL)) { - return NULL; - } - - *prev = b; - prev = &b->next; - } - - /* The end of FastCGI params. */ - p = nxt_cpymem(b->mem.free, nxt_fastcgi_params_record, 8); - - /* The end of FastCGI stdin. */ - b->mem.free = nxt_cpymem(p, nxt_fastcgi_stdin_record, 8); - - nxt_thread_log_debug("\"%*s\"", b->mem.free - b->mem.pos, b->mem.pos); - - return req; -} - - -static nxt_int_t -nxt_fastcgi_next_param(nxt_fastcgi_source_t *fs, nxt_fastcgi_param_t *param) -{ - nxt_int_t ret; - - enum { - sw_name_length = 0, - sw_value_length, - sw_name, - sw_value, - }; - - switch (fs->state) { - - case sw_name_length: - ret = fs->request_create(fs); - - if (nxt_slow_path(ret != NXT_OK)) { - return ret; - } - - nxt_thread_log_debug("fastcgi param \"%V: %V\"", - &fs->u.request.name, &fs->u.request.value); - - fs->state = sw_value_length; - param->buf = param->length; - param->len = nxt_fastcgi_param_length(param->length, - fs->u.request.name.len); - break; - - case sw_value_length: - fs->state = sw_name; - param->buf = param->length; - param->len = nxt_fastcgi_param_length(param->length, - fs->u.request.value.len); - break; - - case sw_name: - fs->state = sw_value; - param->buf = fs->u.request.name.data; - param->len = fs->u.request.name.len; - break; - - case sw_value: - fs->state = sw_name_length; - param->buf = fs->u.request.value.data; - param->len = fs->u.request.value.len; - break; - } - - return NXT_OK; -} - - -static void -nxt_fastcgi_source_record_filter(nxt_task_t *task, void *obj, void *data) -{ - size_t size; - u_char *p; - nxt_buf_t *b, *in; - nxt_fastcgi_source_t *fs; - nxt_fastcgi_source_record_t *fsr; - - fsr = obj; - in = data; - - nxt_debug(task, "fastcgi source record filter"); - - if (nxt_slow_path(fsr->parse.done)) { - return; - } - - nxt_fastcgi_record_parse(task, &fsr->parse, in); - - fs = nxt_container_of(fsr, nxt_fastcgi_source_t, record); - - if (fsr->parse.error) { - nxt_fastcgi_source_fail(task, fs); - return; - } - - if (fsr->parse.fastcgi_error) { - /* - * Output all parsed before a FastCGI record error and close upstream. - */ - nxt_thread_current_work_queue_add(task->thread, - nxt_fastcgi_source_record_error, - task, fs, NULL); - } - - /* Log FastCGI stderr output. */ - - for (b = fsr->parse.out[1]; b != NULL; b = b->next) { - - for (p = b->mem.free - 1; p >= b->mem.pos; p--) { - if (*p != '\r' && *p != '\n') { - break; - } - } - - size = (p + 1) - b->mem.pos; - - if (size != 0) { - nxt_log(task, NXT_LOG_ERR, - "upstream sent in FastCGI stderr: \"%*s\"", - size, b->mem.pos); - } - - b->completion_handler(task, b, b->parent); - } - - /* Process FastCGI stdout output. */ - - if (fsr->parse.out[0] != NULL) { - nxt_source_filter(task->thread, fs->upstream->work_queue, task, - &fsr->next, fsr->parse.out[0]); - } -} - - -static void -nxt_fastcgi_source_record_error(nxt_task_t *task, void *obj, void *data) -{ - nxt_fastcgi_source_t *fs; - - fs = obj; - - nxt_fastcgi_source_fail(task, fs); -} - - -static void -nxt_fastcgi_source_header_filter(nxt_task_t *task, void *obj, void *data) -{ - nxt_int_t ret; - nxt_buf_t *b; - nxt_fastcgi_source_t *fs; - - fs = obj; - b = data; - - do { - nxt_debug(task, "fastcgi source header filter"); - - if (nxt_slow_path(nxt_buf_is_sync(b))) { - nxt_fastcgi_source_sync_buffer(task, fs, b); - return; - } - - for ( ;; ) { - ret = nxt_http_split_header_parse(&fs->u.header, &b->mem); - - if (nxt_slow_path(ret != NXT_OK)) { - break; - } - - ret = nxt_fastcgi_source_header_process(task, fs); - - if (nxt_slow_path(ret != NXT_OK)) { - break; - } - } - - if (nxt_fast_path(ret == NXT_DONE)) { - nxt_debug(task, "fastcgi source header done"); - nxt_fastcgi_source_header_ready(fs, b); - return; - } - - if (nxt_fast_path(ret != NXT_AGAIN)) { - - if (ret != NXT_ERROR) { - /* n == NXT_DECLINED: "\r" is not followed by "\n" */ - nxt_log(task, NXT_LOG_ERR, - "upstream sent invalid header line: \"%*s\\r...\"", - fs->u.header.parse.header_end - - fs->u.header.parse.header_name_start, - fs->u.header.parse.header_name_start); - } - - /* ret == NXT_ERROR */ - - nxt_fastcgi_source_fail(task, fs); - return; - } - - b = b->next; - - } while (b != NULL); -} - - -static void -nxt_fastcgi_source_sync_buffer(nxt_task_t *task, nxt_fastcgi_source_t *fs, - nxt_buf_t *b) -{ - if (nxt_buf_is_last(b)) { - nxt_log(task, NXT_LOG_ERR, "upstream closed prematurely connection"); - - } else { - nxt_log(task, NXT_LOG_ERR, "%ui buffers %uz each are not " - "enough to process upstream response header", - fs->upstream->buffers.max, fs->upstream->buffers.size); - } - - /* The stream source sends only the last and the nobuf sync buffer. */ - - nxt_fastcgi_source_fail(task, fs); -} - - -static nxt_int_t -nxt_fastcgi_source_header_process(nxt_task_t *task, nxt_fastcgi_source_t *fs) -{ - size_t len; - nxt_name_value_t *nv; - nxt_lvlhsh_query_t lhq; - nxt_http_header_parse_t *hp; - nxt_upstream_name_value_t *unv; - - hp = &fs->u.header.parse; - - len = hp->header_name_end - hp->header_name_start; - - if (len > 255) { - nxt_log(task, NXT_LOG_INFO, - "upstream sent too long header field name: \"%*s\"", - len, hp->header_name_start); - return NXT_ERROR; - } - - nv = nxt_list_add(fs->header_in.list); - if (nxt_slow_path(nv == NULL)) { - return NXT_ERROR; - } - - nv->hash = hp->header_hash; - nv->skip = 0; - nv->name_len = len; - nv->name_start = hp->header_name_start; - nv->value_len = hp->header_end - hp->header_start; - nv->value_start = hp->header_start; - - nxt_debug(task, "http header: \"%*s: %*s\"", - nv->name_len, nv->name_start, nv->value_len, nv->value_start); - - lhq.key_hash = nv->hash; - lhq.key.len = nv->name_len; - lhq.key.data = nv->name_start; - lhq.proto = &nxt_upstream_header_hash_proto; - - if (nxt_lvlhsh_find(&fs->header_in.hash, &lhq) == NXT_OK) { - unv = lhq.value; - - if (unv->handler(fs->upstream, nv) == NXT_OK) { - return NXT_ERROR; - } - } - - return NXT_OK; -} - - -static const nxt_upstream_name_value_t nxt_fastcgi_source_headers[] - nxt_aligned(32) = -{ - { nxt_fastcgi_source_status, - nxt_upstream_name_value("status") }, - - { nxt_fastcgi_source_content_length, - nxt_upstream_name_value("content-length") }, -}; - - -nxt_int_t -nxt_fastcgi_source_hash_create(nxt_mp_t *mp, nxt_lvlhsh_t *lh) -{ - return nxt_upstream_header_hash_add(mp, lh, nxt_fastcgi_source_headers, - nxt_nitems(nxt_fastcgi_source_headers)); -} - - -static nxt_int_t -nxt_fastcgi_source_status(nxt_upstream_source_t *us, nxt_name_value_t *nv) -{ - nxt_int_t n; - nxt_str_t s; - nxt_fastcgi_source_t *fs; - - s.len = nv->value_len; - s.data = nv->value_start; - - n = nxt_str_int_parse(&s); - - if (nxt_fast_path(n > 0)) { - fs = us->protocol_source; - fs->header_in.status = n; - return NXT_OK; - } - - return NXT_ERROR; -} - - -static nxt_int_t -nxt_fastcgi_source_content_length(nxt_upstream_source_t *us, - nxt_name_value_t *nv) -{ - nxt_off_t length; - nxt_fastcgi_source_t *fs; - - length = nxt_off_t_parse(nv->value_start, nv->value_len); - - if (nxt_fast_path(length > 0)) { - fs = us->protocol_source; - fs->header_in.content_length = length; - return NXT_OK; - } - - return NXT_ERROR; -} - - -static void -nxt_fastcgi_source_header_ready(nxt_fastcgi_source_t *fs, nxt_buf_t *b) -{ - /* - * Change the FastCGI source filter chain: - * stream source | FastCGI record filter | FastCGI body filter - */ - fs->record.next.filter = nxt_fastcgi_source_body_filter; - - if (nxt_buf_mem_used_size(&b->mem) != 0) { - fs->rest = b; - } - - if (fs->header_in.status == 0) { - /* The "200 OK" status by default. */ - fs->header_in.status = 200; - } - - fs->upstream->state->ready_handler(fs); -} - - -/* - * The FastCGI source body filter accumulates first body buffers before the next - * filter will be established and sets completion handler for the last buffer. - */ - -static void -nxt_fastcgi_source_body_filter(nxt_task_t *task, void *obj, void *data) -{ - nxt_buf_t *b, *in; - nxt_fastcgi_source_t *fs; - - fs = obj; - in = data; - - nxt_debug(task, "fastcgi source body filter"); - - for (b = in; b != NULL; b = b->next) { - - if (nxt_buf_is_last(b)) { - b->data = fs->upstream->data; - b->completion_handler = fs->upstream->state->completion_handler; - } - } - - if (fs->next != NULL) { - nxt_source_filter(task->thread, fs->upstream->work_queue, task, - fs->next, in); - return; - } - - nxt_buf_chain_add(&fs->rest, in); -} - - -static nxt_buf_t * -nxt_fastcgi_source_last_buf(nxt_fastcgi_parse_t *fp) -{ - nxt_buf_t *b; - nxt_fastcgi_source_t *fs; - - fs = fp->data; - - b = nxt_buf_sync_alloc(fp->mem_pool, NXT_BUF_SYNC_LAST); - - if (nxt_fast_path(b != NULL)) { - b->data = fs->upstream->data; - b->completion_handler = fs->upstream->state->completion_handler; - } - - return b; -} - - -static void -nxt_fastcgi_source_error(nxt_task_t *task, nxt_stream_source_t *stream) -{ - nxt_fastcgi_source_t *fs; - - nxt_thread_log_debug("fastcgi source error"); - - fs = stream->upstream->protocol_source; - - nxt_fastcgi_source_fail(task, fs); -} - - -static void -nxt_fastcgi_source_fail(nxt_task_t *task, nxt_fastcgi_source_t *fs) -{ - nxt_debug(task, "fastcgi source fail"); - - /* TODO: fail, next upstream, or bad gateway */ - - fs->upstream->state->error_handler(task, fs, NULL); -} diff --git a/src/nxt_fastcgi_source.h b/src/nxt_fastcgi_source.h deleted file mode 100644 index 979e962b..00000000 --- a/src/nxt_fastcgi_source.h +++ /dev/null @@ -1,93 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_FASTCGI_SOURCE_H_INCLUDED_ -#define _NXT_FASTCGI_SOURCE_H_INCLUDED_ - - -#define NXT_FASTCGI_BEGIN_REQUEST 1 -#define NXT_FASTCGI_ABORT_REQUEST 2 -#define NXT_FASTCGI_END_REQUEST 3 -#define NXT_FASTCGI_PARAMS 4 -#define NXT_FASTCGI_STDIN 5 -#define NXT_FASTCGI_STDOUT 6 -#define NXT_FASTCGI_STDERR 7 -#define NXT_FASTCGI_DATA 8 - - -typedef struct nxt_fastcgi_parse_s nxt_fastcgi_parse_t; - -struct nxt_fastcgi_parse_s { - u_char *pos; - - uint16_t length; /* 16 bits */ - uint8_t padding; - uint8_t type; - - uint8_t state; - uint8_t fastcgi_error; /* 1 bit */ - uint8_t error; /* 1 bit */ - uint8_t done; /* 1 bit */ - - /* FastCGI stdout and stderr buffer chains. */ - nxt_buf_t *out[2]; - - nxt_buf_t *(*last_buf)(nxt_fastcgi_parse_t *fp); - void *data; - nxt_mp_t *mem_pool; -}; - - -typedef struct { - nxt_fastcgi_parse_t parse; - nxt_source_hook_t next; -} nxt_fastcgi_source_record_t; - - -typedef struct { - nxt_str_t name; - nxt_str_t value; - uintptr_t data[3]; -} nxt_fastcgi_source_request_t; - - -typedef struct nxt_fastcgi_source_s nxt_fastcgi_source_t; -typedef nxt_int_t (*nxt_fastcgi_source_request_create_t)( - nxt_fastcgi_source_t *fs); - - -struct nxt_fastcgi_source_s { - nxt_source_hook_t query; - nxt_source_hook_t *next; - - nxt_upstream_source_t *upstream; - - nxt_fastcgi_source_request_create_t request_create; - - nxt_upstream_header_in_t header_in; - - nxt_buf_t *rest; - - uint32_t state; /* 2 bits */ - - nxt_fastcgi_source_record_t record; - - union { - nxt_fastcgi_source_request_t request; - } u; -}; - - -NXT_EXPORT void nxt_fastcgi_source_handler(nxt_task_t *task, - nxt_upstream_source_t *us, - nxt_fastcgi_source_request_create_t request_create); -NXT_EXPORT nxt_int_t nxt_fastcgi_source_hash_create(nxt_mp_t *mp, - nxt_lvlhsh_t *lh); -void nxt_fastcgi_record_parse(nxt_task_t *task, nxt_fastcgi_parse_t *fp, - nxt_buf_t *in); - - -#endif /* _NXT_FASTCGI_SOURCE_H_INCLUDED_ */ diff --git a/src/nxt_file_cache.c b/src/nxt_file_cache.c deleted file mode 100644 index 3af3c0c5..00000000 --- a/src/nxt_file_cache.c +++ /dev/null @@ -1,508 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -static nxt_int_t nxt_file_cache_lvlhsh_test(nxt_lvlhsh_key_t *hkey, void *data); -static nxt_work_handler_t nxt_file_cache_query_locked(nxt_file_cache_t *cache, - nxt_file_cache_query_t *q, nxt_lvlhsh_key_t *hkey); -static nxt_work_handler_t nxt_file_cache_node_hold(nxt_file_cache_t *cache, - nxt_file_cache_query_t *q, nxt_lvlhsh_key_t *hkey); -static nxt_work_handler_t nxt_file_cache_node_test(nxt_file_cache_t *cache, - nxt_file_cache_query_t *q); - -static void nxt_file_cache_wait_handler(void *data); -static void nxt_file_cache_timeout_handler(nxt_event_timer_t *ev); -static void nxt_file_cache_wake_handler(void *data); - -static nxt_file_cache_node_t *nxt_file_cache_node_alloc(nxt_cache_t *cache); -static void nxt_file_cache_node_free(nxt_file_cache_t *cache, - nxt_file_cache_node_t *node, nxt_bool_t fast); -static nxt_file_cache_query_wait_t *nxt_file_cache_query_wait_alloc( - nxt_file_cache_t *cache, nxt_bool_t *fast); -static void nxt_file_cache_query_wait_free(nxt_file_cache_t *cache, - nxt_file_cache_query_wait_t *qw); -static void nxt_file_cache_lock(nxt_file_cache_t *cache); -static void nxt_file_cache_unlock(nxt_file_cache_t *cache); - - -void -nxt_file_cache_init(nxt_cache_t *cache) -{ - static const nxt_lvlhsh_ctx_t ctx = { - nxt_file_cache_lvlhsh_test, - nxt_lvlhsh_alloc, - nxt_lvlhsh_free, - 0, - }; - - /* lvlhsh with large first level. */ - cache->lvlhsh.shift[1] = 10; - - cache->lvlhsh.ctx = &ctx; - - cache->start_time = nxt_thread_time(); -} - - -static nxt_int_t -nxt_file_cache_lvlhsh_test(nxt_lvlhsh_key_t *hkey, void *data) -{ - nxt_file_cache_node_t *node; - - node = data; - - if (nxt_strmem_eq(&hkey->key, node->key_data, node->key_len)) { - return NXT_OK; - } - - return NXT_DECLINED; -} - - -void -nxt_file_cache_query(nxt_file_cache_t *cache, nxt_file_cache_query_t *q) -{ - nxt_lvlhsh_key_t hkey; - nxt_work_handler_t handler; - - if (cache != NULL) { - hkey.key.len = q->key_len; - hkey.key.data = q->key_data; - hkey.key_hash = nxt_murmur_hash2(q->key_data, q->key_len); - hkey.replace = 0; - - nxt_file_cache_lock(cache); - - handler = nxt_file_cache_query_locked(cache, q, &hkey); - - nxt_file_cache_unlock(cache); - - } else { - handler = q->state->nocache_handler; - } - - handler(q); -} - - -static nxt_work_handler_t -nxt_file_cache_query_locked(nxt_file_cache_t *cache, nxt_file_cache_query_t *q, - nxt_lvlhsh_key_t *hkey) -{ - nxt_int_t ret; - nxt_bool_t fast; - nxt_work_handler_t handler; - nxt_file_cache_node_t *node, *sentinel; - nxt_file_cache_query_wait_t *qw; - nxt_file_cache_query_state_t *state; - - state = q->state; - sentinel = nxt_file_cache_node_alloc(cache); - - if (nxt_slow_path(sentinel == NULL)) { - return state->error_handler; - } - - sentinel->key_data = q->key_data; - sentinel->key_len = q->key_len; - hkey->value = sentinel; - - /* - * Try to insert an empty sentinel node to hold updating - * process if there is no existent cache node in cache. - */ - - ret = nxt_lvlhsh_insert(&cache->lvlhsh, hkey); - - if (ret == NXT_OK) { - /* The sentinel node was successully added. */ - - q->node = sentinel; - sentinel->updating = 1; - return state->update_handler; - } - - nxt_cache_node_free(cache, sentinel, 1); - - if (ret == NXT_ERROR) { - return state->error_handler; - } - - /* NXT_DECLINED: a cache node exists. */ - - node = hkey->value; - node->count++; - q->node = node; - - handler = nxt_cache_node_test(cache, q); - - if (handler == NULL) { - /* Add the node to a wait queue. */ - - qw = nxt_cache_query_wait_alloc(cache, &fast); - if (nxt_slow_path(qw == NULL)) { - return state->error_handler; - } - - if (!fast) { - /* The node state may be changed during slow allocation. */ - handler = nxt_cache_node_test(cache, q); - - if (handler != NULL) { - nxt_cache_query_wait_free(cache, qw); - return handler; - } - } - - qw->query = q; - qw->next = node->waiting; - qw->busy = 0; - qw->deleted = 0; - qw->pid = nxt_pid; - qw->engine = nxt_thread_event_engine(); - qw->handler = nxt_cache_wake_handler; - qw->cache = cache; - - node->waiting = qw; - - return nxt_cache_wait_handler; - } - - return handler; -} - - -static nxt_work_handler_t -nxt_cache_node_test(nxt_cache_t *cache, nxt_cache_query_t *q) -{ - nxt_time_t expiry; - nxt_cache_node_t *node; - nxt_cache_query_state_t *state; - - q->stale = 0; - state = q->state; - node = q->node; - - expiry = cache->start_time + node->expiry; - - if (nxt_thread_time() < expiry) { - return state->ready_handler; - } - - /* - * A valid stale or empty sentinel cache node. - * The sentinel node can be only in updating state. - */ - - if (node->updating) { - - if (node->expiry != 0) { - /* A valid stale cache node. */ - - q->stale = 1; - - if (q->use_stale) { - return state->stale_handler; - } - } - - return NULL; - } - - /* A valid stale cache node is not being updated now. */ - - q->stale = 1; - - if (q->use_stale) { - - if (q->update_stale) { - node->updating = 1; - return state->update_stale_handler; - } - - return state->stale_handler; - } - - node->updating = 1; - return state->update_handler; -} - - -static void -nxt_cache_wait_handler(void *data) -{ - nxt_thread_t *thr; - nxt_event_timer_t *ev; - nxt_cache_query_t *q; - - q = data; - - if (&q->timeout == 0) { - return; - } - - ev = &q->timer; - - if (!nxt_event_timer_is_set(ev)) { - thr = nxt_thread(); - ev->log = thr->log; - ev->handler = nxt_cache_timeout_handler; - ev->data = q; - nxt_event_timer_ident(ev, -1); - - nxt_event_timer_add(thr->engine, ev, q->timeout); - } -} - - -static void -nxt_cache_timeout_handler(nxt_event_timer_t *ev) -{ - nxt_cache_query_t *q; - - q = ev->data; - - q->state->timeout_handler(q); -} - - -static void -nxt_cache_wake_handler(void *data) -{ - nxt_cache_t *cache; - nxt_work_handler_t handler; - nxt_cache_query_t *q; - nxt_cache_query_wait_t *qw; - - qw = data; - q = qw->query; - cache = qw->cache; - - nxt_cache_lock(cache); - - handler = nxt_cache_node_test(cache, q); - - if (handler == NULL) { - /* Wait again. */ - qw->next = q->node->waiting; - q->node->waiting = qw; - } - - nxt_cache_unlock(cache); - - if (handler != NULL) { - nxt_cache_query_wait_free(cache, qw); - } - - handler(q); -} - - -static nxt_cache_node_t * -nxt_cache_node_alloc(nxt_cache_t *cache) -{ - nxt_queue_node_t *qn; - nxt_cache_node_t *node; - - qn = nxt_queue_first(&cache->free_nodes); - - if (nxt_fast_path(qn != nxt_queue_tail(&cache->free_nodes))) { - cache->nfree_nodes--; - nxt_queue_remove(qn); - - node = nxt_queue_node_data(qn, nxt_cache_node_t, queue); - nxt_memzero(node, sizeof(nxt_cache_node_t)); - - return node; - } - - nxt_cache_unlock(cache); - - node = cache->alloc(cache->data, sizeof(nxt_cache_node_t)); - - nxt_cache_lock(cache); - - return node; -} - - -static void -nxt_cache_node_free(nxt_cache_t *cache, nxt_cache_node_t *node, nxt_bool_t fast) -{ - if (fast || cache->nfree_nodes < 32) { - nxt_queue_insert_head(&cache->free_nodes, &node->queue); - cache->nfree_nodes++; - return; - } - - nxt_cache_unlock(cache); - - cache->free(cache->data, node); - - nxt_cache_lock(cache); -} - - -static nxt_cache_query_wait_t * -nxt_cache_query_wait_alloc(nxt_cache_t *cache, nxt_bool_t *fast) -{ - nxt_cache_query_wait_t *qw; - - qw = cache->free_query_wait; - - if (nxt_fast_path(qw != NULL)) { - cache->free_query_wait = qw->next; - cache->nfree_query_wait--; - - *fast = 1; - return qw; - } - - nxt_cache_unlock(cache); - - qw = cache->alloc(cache->data, sizeof(nxt_cache_query_wait_t)); - *fast = 0; - - nxt_cache_lock(cache); - - return qw; -} - - -static void -nxt_cache_query_wait_free(nxt_cache_t *cache, nxt_cache_query_wait_t *qw) -{ - if (cache->nfree_query_wait < 32) { - qw->next = cache->free_query_wait; - cache->free_query_wait = qw; - cache->nfree_query_wait++; - return; - } - - nxt_cache_unlock(cache); - - cache->free(cache->data, qw); - - nxt_cache_lock(cache); -} - - -#if 0 - -nxt_int_t -nxt_cache_update(nxt_cache_t *cache, nxt_cache_node_t *node) -{ - nxt_lvlhsh_key_t hkey; - - if (node->expiry == 0) { - /* An empty sentinel node. */ - nxt_cache_release(cache, node); - return; - } - - hkey.key.len = node->key_len; - hkey.key.data = node->key_data; - hkey.key_hash = nxt_murmur_hash2(node->key_data, node->key_len); - hkey.replace = 1; - hkey.value = node; - - node->count = 1; - - if (nxt_lvlhsh_insert(&cache->lvlhsh, &hkey) != NXT_OK) { - return NXT_ERROR; - } - - node = hkey.value; - - if (node != NULL) { - if (node->count != 0) { - node->delete = 1; - - } else { - // delete cache node - } - } - - return NXT_OK; -} - -#endif - - -void -nxt_cache_node_release(nxt_cache_t *cache, nxt_cache_node_t *node) -{ - nxt_bool_t delete; - - nxt_cache_lock(cache); - - delete = nxt_cache_node_release_locked(cache, node); - - nxt_cache_unlock(cache); - - if (delete) { - nxt_thread_work_queue_add(cache->delete_handler, node); - } -} - - -nxt_bool_t -nxt_cache_node_release_locked(nxt_cache_t *cache, nxt_cache_node_t *node) -{ -#if 0 - nxt_lvlhsh_key_t hkey; -#endif - - node->count--; - - if (node->count != 0) { - return 0; - } - - if (!node->deleted) { - /* - * A cache node is locked whilst its count is non zero. - * To minimize number of operations the node's place in expiry - * queue can be updated only if the node is not currently used. - */ - node->accessed = nxt_thread_time() - cache->start_time; - - nxt_queue_remove(&node->queue); - nxt_queue_insert_head(&cache->expiry_queue, &node->queue); - - return 0; - } - -#if 0 - hkey.key.len = node->key_len; - hkey.key.data = node->key_data; - hkey.key_hash = nxt_murmur_hash2(node->key_data, node->key_len); - - nxt_lvlhsh_delete(&cache->lvlhsh, &hkey); -#endif - - return 1; -} - - -static void -nxt_file_cache_lock(nxt_file_cache_t *cache) -{ - if (cache->shared) { - nxt_thread_spin_lock(&cache->lock); - } -} - - -static void -nxt_file_cache_unlock(nxt_file_cache_t *cache) -{ - if (cache->shared) { - nxt_thread_spin_unlock(&cache->lock); - } -} diff --git a/src/nxt_http_source.c b/src/nxt_http_source.c deleted file mode 100644 index 889dcd08..00000000 --- a/src/nxt_http_source.c +++ /dev/null @@ -1,629 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -typedef struct { - nxt_http_chunk_parse_t parse; - nxt_source_hook_t next; -} nxt_http_source_chunk_t; - - -static nxt_buf_t *nxt_http_source_request_create(nxt_http_source_t *hs); - -static void nxt_http_source_status_filter(nxt_task_t *task, void *obj, - void *data); -static void nxt_http_source_header_filter(nxt_task_t *task, void *obj, - void *data); - -static nxt_int_t nxt_http_source_header_line_process(nxt_http_source_t *hs); -static nxt_int_t nxt_http_source_content_length(nxt_upstream_source_t *us, - nxt_name_value_t *nv); -static nxt_int_t nxt_http_source_transfer_encoding(nxt_upstream_source_t *us, - nxt_name_value_t *nv); - -static void nxt_http_source_header_ready(nxt_task_t *task, - nxt_http_source_t *hs, nxt_buf_t *rest); -static void nxt_http_source_chunk_filter(nxt_task_t *task, void *obj, - void *data); -static void nxt_http_source_chunk_error(nxt_task_t *task, void *obj, - void *data); -static void nxt_http_source_body_filter(nxt_task_t *task, void *obj, - void *data); - -static void nxt_http_source_sync_buffer(nxt_task_t *task, nxt_http_source_t *hs, - nxt_buf_t *b); -static void nxt_http_source_error(nxt_task_t *task, - nxt_stream_source_t *stream); -static void nxt_http_source_fail(nxt_task_t *task, nxt_http_source_t *hs); -static void nxt_http_source_message(const char *msg, size_t len, u_char *p); - - -void -nxt_http_source_handler(nxt_task_t *task, nxt_upstream_source_t *us, - nxt_http_source_request_create_t request_create) -{ - nxt_http_source_t *hs; - nxt_stream_source_t *stream; - - hs = nxt_mp_zget(us->buffers.mem_pool, sizeof(nxt_http_source_t)); - if (nxt_slow_path(hs == NULL)) { - goto fail; - } - - us->protocol_source = hs; - - hs->header_in.list = nxt_list_create(us->buffers.mem_pool, 8, - sizeof(nxt_name_value_t)); - if (nxt_slow_path(hs->header_in.list == NULL)) { - goto fail; - } - - hs->header_in.hash = us->header_hash; - hs->upstream = us; - hs->request_create = request_create; - - stream = us->stream; - - if (stream == NULL) { - stream = nxt_mp_zget(us->buffers.mem_pool, sizeof(nxt_stream_source_t)); - if (nxt_slow_path(stream == NULL)) { - goto fail; - } - - us->stream = stream; - stream->upstream = us; - - } else { - nxt_memzero(stream, sizeof(nxt_stream_source_t)); - } - - /* - * Create the HTTP source filter chain: - * stream source | HTTP status line filter - */ - stream->next = &hs->query; - stream->error_handler = nxt_http_source_error; - - hs->query.context = hs; - hs->query.filter = nxt_http_source_status_filter; - - hs->header_in.content_length = -1; - - stream->out = nxt_http_source_request_create(hs); - - if (nxt_fast_path(stream->out != NULL)) { - nxt_memzero(&hs->u.status_parse, sizeof(nxt_http_status_parse_t)); - - nxt_stream_source_connect(task, stream); - return; - } - -fail: - - nxt_http_source_fail(task, hs); -} - - -nxt_inline u_char * -nxt_http_source_copy(u_char *p, nxt_str_t *src, size_t len) -{ - u_char *s; - - if (nxt_fast_path(len >= src->len)) { - len = src->len; - src->len = 0; - - } else { - src->len -= len; - } - - s = src->data; - src->data += len; - - return nxt_cpymem(p, s, len); -} - - -static nxt_buf_t * -nxt_http_source_request_create(nxt_http_source_t *hs) -{ - nxt_int_t ret; - nxt_buf_t *b, *req, **prev; - - nxt_thread_log_debug("http source create request"); - - prev = &req; - -new_buffer: - - ret = nxt_buf_pool_mem_alloc(&hs->upstream->buffers, 0); - if (nxt_slow_path(ret != NXT_OK)) { - return NULL; - } - - b = hs->upstream->buffers.current; - hs->upstream->buffers.current = NULL; - - *prev = b; - prev = &b->next; - - for ( ;; ) { - ret = hs->request_create(hs); - - if (nxt_fast_path(ret == NXT_OK)) { - b->mem.free = nxt_http_source_copy(b->mem.free, &hs->u.request.copy, - b->mem.end - b->mem.free); - - if (nxt_fast_path(hs->u.request.copy.len == 0)) { - continue; - } - - nxt_thread_log_debug("\"%*s\"", b->mem.free - b->mem.pos, - b->mem.pos); - - goto new_buffer; - } - - if (nxt_slow_path(ret == NXT_ERROR)) { - return NULL; - } - - /* ret == NXT_DONE */ - break; - } - - nxt_thread_log_debug("\"%*s\"", b->mem.free - b->mem.pos, b->mem.pos); - - return req; -} - - -static void -nxt_http_source_status_filter(nxt_task_t *task, void *obj, void *data) -{ - nxt_int_t ret; - nxt_buf_t *b; - nxt_http_source_t *hs; - - hs = obj; - b = data; - - /* - * No cycle over buffer chain is required since at - * start the stream source passes buffers one at a time. - */ - - nxt_debug(task, "http source status filter"); - - if (nxt_slow_path(nxt_buf_is_sync(b))) { - nxt_http_source_sync_buffer(task, hs, b); - return; - } - - ret = nxt_http_status_parse(&hs->u.status_parse, &b->mem); - - if (nxt_fast_path(ret == NXT_OK)) { - /* - * Change the HTTP source filter chain: - * stream source | HTTP header filter - */ - hs->query.filter = nxt_http_source_header_filter; - - nxt_debug(task, "upstream status: \"%*s\"", - hs->u.status_parse.end - b->mem.start, b->mem.start); - - hs->header_in.status = hs->u.status_parse.code; - - nxt_debug(task, "upstream version:%d status:%uD \"%*s\"", - hs->u.status_parse.http_version, - hs->u.status_parse.code, - hs->u.status_parse.end - hs->u.status_parse.start, - hs->u.status_parse.start); - - nxt_memzero(&hs->u.header, sizeof(nxt_http_split_header_parse_t)); - hs->u.header.mem_pool = hs->upstream->buffers.mem_pool; - - nxt_http_source_header_filter(task, hs, b); - return; - } - - if (nxt_slow_path(ret == NXT_ERROR)) { - /* HTTP/0.9 response. */ - hs->header_in.status = 200; - nxt_http_source_header_ready(task, hs, b); - return; - } - - /* ret == NXT_AGAIN */ - - /* - * b->mem.pos is always equal to b->mem.end because b is a buffer - * which points to a response part read by the stream source. - * However, since the stream source is an immediate source of the - * status filter, b->parent is a buffer the stream source reads in. - */ - if (b->parent->mem.pos == b->parent->mem.end) { - nxt_http_source_message("upstream sent too long status line: \"%*s\"", - b->mem.pos - b->mem.start, b->mem.start); - - nxt_http_source_fail(task, hs); - } -} - - -static void -nxt_http_source_header_filter(nxt_task_t *task, void *obj, void *data) -{ - nxt_int_t ret; - nxt_buf_t *b; - nxt_http_source_t *hs; - - hs = obj; - b = data; - - /* - * No cycle over buffer chain is required since at - * start the stream source passes buffers one at a time. - */ - - nxt_debug(task, "http source header filter"); - - if (nxt_slow_path(nxt_buf_is_sync(b))) { - nxt_http_source_sync_buffer(task, hs, b); - return; - } - - for ( ;; ) { - ret = nxt_http_split_header_parse(&hs->u.header, &b->mem); - - if (nxt_slow_path(ret != NXT_OK)) { - break; - } - - ret = nxt_http_source_header_line_process(hs); - - if (nxt_slow_path(ret != NXT_OK)) { - break; - } - } - - if (nxt_fast_path(ret == NXT_DONE)) { - nxt_debug(task, "http source header done"); - nxt_http_source_header_ready(task, hs, b); - return; - } - - if (nxt_fast_path(ret == NXT_AGAIN)) { - return; - } - - if (ret != NXT_ERROR) { - /* ret == NXT_DECLINED: "\r" is not followed by "\n" */ - nxt_log(task, NXT_LOG_ERR, - "upstream sent invalid header line: \"%*s\\r...\"", - hs->u.header.parse.header_end - - hs->u.header.parse.header_name_start, - hs->u.header.parse.header_name_start); - } - - /* ret == NXT_ERROR */ - - nxt_http_source_fail(task, hs); -} - - -static nxt_int_t -nxt_http_source_header_line_process(nxt_http_source_t *hs) -{ - size_t name_len; - nxt_name_value_t *nv; - nxt_lvlhsh_query_t lhq; - nxt_http_header_parse_t *hp; - nxt_upstream_name_value_t *unv; - - hp = &hs->u.header.parse; - - name_len = hp->header_name_end - hp->header_name_start; - - if (name_len > 255) { - nxt_http_source_message("upstream sent too long header field name: " - "\"%*s\"", name_len, hp->header_name_start); - return NXT_ERROR; - } - - nv = nxt_list_add(hs->header_in.list); - if (nxt_slow_path(nv == NULL)) { - return NXT_ERROR; - } - - nv->hash = hp->header_hash; - nv->skip = 0; - nv->name_len = name_len; - nv->name_start = hp->header_name_start; - nv->value_len = hp->header_end - hp->header_start; - nv->value_start = hp->header_start; - - nxt_thread_log_debug("upstream header: \"%*s: %*s\"", - nv->name_len, nv->name_start, - nv->value_len, nv->value_start); - - lhq.key_hash = nv->hash; - lhq.key.len = nv->name_len; - lhq.key.data = nv->name_start; - lhq.proto = &nxt_upstream_header_hash_proto; - - if (nxt_lvlhsh_find(&hs->header_in.hash, &lhq) == NXT_OK) { - unv = lhq.value; - - if (unv->handler(hs->upstream, nv) != NXT_OK) { - return NXT_ERROR; - } - } - - return NXT_OK; -} - - -static const nxt_upstream_name_value_t nxt_http_source_headers[] - nxt_aligned(32) = -{ - { nxt_http_source_content_length, - nxt_upstream_name_value("content-length") }, - - { nxt_http_source_transfer_encoding, - nxt_upstream_name_value("transfer-encoding") }, -}; - - -nxt_int_t -nxt_http_source_hash_create(nxt_mp_t *mp, nxt_lvlhsh_t *lh) -{ - return nxt_upstream_header_hash_add(mp, lh, nxt_http_source_headers, - nxt_nitems(nxt_http_source_headers)); -} - - -static nxt_int_t -nxt_http_source_content_length(nxt_upstream_source_t *us, nxt_name_value_t *nv) -{ - nxt_off_t length; - nxt_http_source_t *hs; - - length = nxt_off_t_parse(nv->value_start, nv->value_len); - - if (nxt_fast_path(length > 0)) { - hs = us->protocol_source; - hs->header_in.content_length = length; - return NXT_OK; - } - - return NXT_ERROR; -} - - -static nxt_int_t -nxt_http_source_transfer_encoding(nxt_upstream_source_t *us, - nxt_name_value_t *nv) -{ - u_char *end; - nxt_http_source_t *hs; - - end = nv->value_start + nv->value_len; - - if (nxt_memcasestrn(nv->value_start, end, "chunked", 7) != NULL) { - hs = us->protocol_source; - hs->chunked = 1; - } - - return NXT_OK; -} - - -static void -nxt_http_source_header_ready(nxt_task_t *task, nxt_http_source_t *hs, - nxt_buf_t *rest) -{ - nxt_buf_t *b; - nxt_upstream_source_t *us; - nxt_http_source_chunk_t *hsc; - - us = hs->upstream; - - /* Free buffers used for request header. */ - - for (b = us->stream->out; b != NULL; b = b->next) { - nxt_buf_pool_free(&us->buffers, b); - } - - if (nxt_fast_path(nxt_buf_pool_available(&us->buffers))) { - - if (hs->chunked) { - hsc = nxt_mp_zalloc(hs->upstream->buffers.mem_pool, - sizeof(nxt_http_source_chunk_t)); - if (nxt_slow_path(hsc == NULL)) { - goto fail; - } - - /* - * Change the HTTP source filter chain: - * stream source | chunk filter | HTTP body filter - */ - hs->query.context = hsc; - hs->query.filter = nxt_http_source_chunk_filter; - - hsc->next.context = hs; - hsc->next.filter = nxt_http_source_body_filter; - - hsc->parse.mem_pool = hs->upstream->buffers.mem_pool; - - if (nxt_buf_mem_used_size(&rest->mem) != 0) { - hs->rest = nxt_http_chunk_parse(task, &hsc->parse, rest); - - if (nxt_slow_path(hs->rest == NULL)) { - goto fail; - } - } - - } else { - /* - * Change the HTTP source filter chain: - * stream source | HTTP body filter - */ - hs->query.filter = nxt_http_source_body_filter; - - if (nxt_buf_mem_used_size(&rest->mem) != 0) { - hs->rest = rest; - } - } - - hs->upstream->state->ready_handler(hs); - return; - } - - nxt_thread_log_error(NXT_LOG_ERR, "%d buffers %uDK each " - "are not enough to read upstream response", - us->buffers.max, us->buffers.size / 1024); -fail: - - nxt_http_source_fail(task, hs); -} - - -static void -nxt_http_source_chunk_filter(nxt_task_t *task, void *obj, void *data) -{ - nxt_buf_t *b; - nxt_http_source_t *hs; - nxt_http_source_chunk_t *hsc; - - hsc = obj; - b = data; - - nxt_debug(task, "http source chunk filter"); - - b = nxt_http_chunk_parse(task, &hsc->parse, b); - - hs = hsc->next.context; - - if (hsc->parse.error) { - nxt_http_source_fail(task, hs); - return; - } - - if (hsc->parse.chunk_error) { - /* Output all parsed before a chunk error and close upstream. */ - nxt_thread_current_work_queue_add(task->thread, - nxt_http_source_chunk_error, - task, hs, NULL); - } - - if (b != NULL) { - nxt_source_filter(task->thread, hs->upstream->work_queue, task, - &hsc->next, b); - } -} - - -static void -nxt_http_source_chunk_error(nxt_task_t *task, void *obj, void *data) -{ - nxt_http_source_t *hs; - - hs = obj; - - nxt_http_source_fail(task, hs); -} - - -/* - * The HTTP source body filter accumulates first body buffers before the next - * filter will be established and sets completion handler for the last buffer. - */ - -static void -nxt_http_source_body_filter(nxt_task_t *task, void *obj, void *data) -{ - nxt_buf_t *b, *in; - nxt_http_source_t *hs; - - hs = obj; - in = data; - - nxt_debug(task, "http source body filter"); - - for (b = in; b != NULL; b = b->next) { - - if (nxt_buf_is_last(b)) { - b->data = hs->upstream->data; - b->completion_handler = hs->upstream->state->completion_handler; - } - } - - if (hs->next != NULL) { - nxt_source_filter(task->thread, hs->upstream->work_queue, task, - hs->next, in); - return; - } - - nxt_buf_chain_add(&hs->rest, in); -} - - -static void -nxt_http_source_sync_buffer(nxt_task_t *task, nxt_http_source_t *hs, - nxt_buf_t *b) -{ - if (nxt_buf_is_last(b)) { - nxt_log(task, NXT_LOG_ERR, - "upstream closed prematurely connection"); - - } else { - nxt_log(task, NXT_LOG_ERR,"%ui buffers %uz each are not " - "enough to process upstream response header", - hs->upstream->buffers.max, hs->upstream->buffers.size); - } - - /* The stream source sends only the last and the nobuf sync buffer. */ - - nxt_http_source_fail(task, hs); -} - - -static void -nxt_http_source_error(nxt_task_t *task, nxt_stream_source_t *stream) -{ - nxt_http_source_t *hs; - - nxt_thread_log_debug("http source error"); - - hs = stream->next->context; - nxt_http_source_fail(task, hs); -} - - -static void -nxt_http_source_fail(nxt_task_t *task, nxt_http_source_t *hs) -{ - nxt_debug(task, "http source fail"); - - /* TODO: fail, next upstream, or bad gateway */ - - hs->upstream->state->error_handler(task, hs, NULL); -} - - -static void -nxt_http_source_message(const char *msg, size_t len, u_char *p) -{ - if (len > NXT_MAX_ERROR_STR - 300) { - len = NXT_MAX_ERROR_STR - 300; - p[len++] = '.'; p[len++] = '.'; p[len++] = '.'; - } - - nxt_thread_log_error(NXT_LOG_ERR, msg, len, p); -} diff --git a/src/nxt_http_source.h b/src/nxt_http_source.h deleted file mode 100644 index 7cf2876b..00000000 --- a/src/nxt_http_source.h +++ /dev/null @@ -1,47 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_HTTP_SOURCE_H_INCLUDED_ -#define _NXT_HTTP_SOURCE_H_INCLUDED_ - - -typedef struct { - nxt_str_t copy; - uintptr_t data[3]; -} nxt_http_source_request_t; - - -typedef struct nxt_http_source_s nxt_http_source_t; -typedef nxt_int_t (*nxt_http_source_request_create_t)(nxt_http_source_t *hs); - - -struct nxt_http_source_s { - nxt_source_hook_t query; - nxt_source_hook_t *next; - - nxt_upstream_source_t *upstream; - - nxt_http_source_request_create_t request_create; - - nxt_upstream_header_in_t header_in; - - nxt_buf_t *rest; - - uint32_t chunked; /* 1 bit */ - - union { - nxt_http_source_request_t request; - } u; -}; - - -NXT_EXPORT void nxt_http_source_handler(nxt_task_t *task, - nxt_upstream_source_t *us, nxt_http_source_request_create_t request_create); -NXT_EXPORT nxt_int_t nxt_http_source_hash_create(nxt_mp_t *mp, - nxt_lvlhsh_t *lh); - - -#endif /* _NXT_HTTP_SOURCE_H_INCLUDED_ */ diff --git a/src/nxt_job_file.c b/src/nxt_job_file.c deleted file mode 100644 index 675bed2f..00000000 --- a/src/nxt_job_file.c +++ /dev/null @@ -1,302 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - - -#include - - -static void nxt_job_file_open_and_read(nxt_task_t *task, void *obj, void *data); -static nxt_int_t nxt_job_file_open(nxt_job_file_t *jbf); -static nxt_int_t nxt_job_file_info(nxt_job_file_t *jbf); -static nxt_int_t nxt_job_file_mmap(nxt_job_file_t *jbf, size_t size); -static nxt_int_t nxt_job_file_read_data(nxt_job_file_t *jbf, size_t size); -static nxt_int_t nxt_job_file_read_required(nxt_job_file_t *jbf); - - -nxt_job_file_t * -nxt_job_file_create(nxt_mp_t *mp) -{ - nxt_job_file_t *jbf; - - jbf = nxt_job_create(mp, sizeof(nxt_job_file_t)); - - if (nxt_fast_path(jbf != NULL)) { - jbf->file.fd = NXT_FILE_INVALID; - jbf->file.accessed = NXT_FILE_ACCESSED_LONG_AGO; - jbf->read_required = nxt_job_file_read_required; - } - - return jbf; -} - - -void -nxt_job_file_init(nxt_job_file_t *jbf) -{ - nxt_job_init(&jbf->job, sizeof(nxt_job_file_t)); - - jbf->file.fd = NXT_FILE_INVALID; - jbf->file.accessed = NXT_FILE_ACCESSED_LONG_AGO; - jbf->read_required = nxt_job_file_read_required; -} - - -/* - * Must be a function but not a macro, because - * it can be used as function pointer. - */ - -void -nxt_job_file_read(nxt_task_t *task, nxt_job_t *job) -{ - nxt_job_start(task, job, nxt_job_file_open_and_read); -} - - -static void -nxt_job_file_open_and_read(nxt_task_t *task, void *obj, void *data) -{ - size_t size; - nxt_int_t n; - nxt_bool_t read_ahead; - nxt_file_t *file; - nxt_job_file_t *jbf; - nxt_work_handler_t handler; - - jbf = obj; - file = &jbf->file; - - nxt_debug(task, "file job read: \"%FN\"", file->name); - - if (file->fd != NXT_FILE_INVALID && jbf->close_before_open) { - nxt_file_close(file); - file->fd = NXT_FILE_INVALID; - } - - if (file->fd == NXT_FILE_INVALID) { - - switch (nxt_job_file_open(jbf)) { - - case NXT_OK: - break; - - case NXT_DECLINED: - handler = jbf->ready_handler; - goto done; - - default: /* NXT_ERROR */ - handler = jbf->error_handler; - goto done; - } - } - - if (file->size > 0) { - - if (jbf->buffer != NULL) { - size = nxt_buf_mem_size(&jbf->buffer->mem); - size = nxt_min(file->size, (nxt_off_t) size); - read_ahead = nxt_buf_is_mmap(jbf->buffer); - - } else { - size = nxt_min(file->size, 1024 * 1024); - read_ahead = jbf->read_ahead; - } - - if (read_ahead) { - nxt_file_read_ahead(&jbf->file, jbf->offset, size); - } - - if (jbf->buffer != NULL) { - - if (nxt_buf_is_mmap(jbf->buffer)) { - n = nxt_job_file_mmap(jbf, size); - - } else { - n = nxt_job_file_read_data(jbf, size); - } - - if (nxt_slow_path(n != NXT_OK)) { - handler = jbf->error_handler; - goto done; - } - } - } - - if (jbf->offset == file->size) { - jbf->complete = 1; - - if (jbf->close) { - nxt_file_close(file); - file->fd = NXT_FILE_INVALID; - } - } - - nxt_job_return(task, &jbf->job, jbf->ready_handler); - return; - -done: - - if (file->fd != NXT_FILE_INVALID) { - nxt_file_close(file); - file->fd = NXT_FILE_INVALID; - } - - nxt_job_return(task, &jbf->job, handler); -} - - -static nxt_int_t -nxt_job_file_open(nxt_job_file_t *jbf) -{ - nxt_int_t n; - - if (jbf->test_before_open) { - n = nxt_job_file_info(jbf); - - if (n != NXT_OK) { - goto test_directory; - } - - if (jbf->file.type == NXT_FILE_DIRECTORY) { - return NXT_DECLINED; - } - - if (jbf->read_required(jbf) != NXT_OK) { - return NXT_DECLINED; - } - } - - n = nxt_file_open(&jbf->file, NXT_FILE_RDONLY, NXT_FILE_OPEN, 0); - - if (n == NXT_OK) { - n = nxt_job_file_info(jbf); - - if (nxt_fast_path(n == NXT_OK)) { - - if (jbf->file.type == NXT_FILE_DIRECTORY) { - return NXT_DECLINED; - } - - return jbf->read_required(jbf); - } - - return n; - } - -test_directory: - - if (jbf->directory_end != 0 - && jbf->file.error != NXT_ENOTDIR - && jbf->file.error != NXT_ENAMETOOLONG - && jbf->file.error != NXT_EACCES) - { - jbf->file.name[jbf->directory_end] = '\0'; - - return nxt_job_file_info(jbf); - } - - return n; -} - - -static nxt_int_t -nxt_job_file_info(nxt_job_file_t *jbf) -{ - nxt_int_t n; - nxt_file_t *file; - nxt_file_info_t fi; - - file = &jbf->file; - - n = nxt_file_info(file, &fi); - - if (n != NXT_OK) { - return NXT_ERROR; - } - - if (nxt_is_file(&fi)) { - file->type = NXT_FILE_REGULAR; - file->size = nxt_file_size(&fi); - file->mtime = nxt_file_mtime(&fi); - - } else if (nxt_is_dir(&fi)) { - file->type = NXT_FILE_DIRECTORY; - file->size = nxt_file_size(&fi); - file->mtime = nxt_file_mtime(&fi); - } - - return NXT_OK; -} - - -static nxt_int_t -nxt_job_file_mmap(nxt_job_file_t *jbf, size_t size) -{ - u_char *p, *end; - static nxt_uint_t n; - - p = nxt_mem_map(NULL, &jbf->buffer->mmap, size, NXT_MEM_MAP_READ, - (NXT_MEM_MAP_FILE | NXT_MEM_MAP_PREFAULT), - jbf->file.fd, jbf->offset); - - if (nxt_fast_path(p != NXT_MEM_MAP_FAILED)) { - - end = p + size; - - jbf->buffer->mem.pos = p; - jbf->buffer->mem.free = end; - jbf->buffer->mem.start = p; - jbf->buffer->mem.end = end; - jbf->buffer->file_end += size; - jbf->offset += size; - - /* - * The mapped pages should be already preloaded in the kernel page - * cache by nxt_file_read_ahead(). Touching them should wire the pages - * in user land memory if mmap() did not do this. Adding to the static - * variable "n" disables the loop elimination during optimization. - */ - n += *p; - - for (p = nxt_align_ptr(p, nxt_pagesize); p < end; p += nxt_pagesize) { - n += *p; - } - - return NXT_OK; - } - - return NXT_ERROR; -} - - -static nxt_int_t -nxt_job_file_read_data(nxt_job_file_t *jbf, size_t size) -{ - ssize_t n; - - n = nxt_file_read(&jbf->file, jbf->buffer->mem.free, size, jbf->offset); - - if (nxt_fast_path(n > 0)) { - - jbf->buffer->mem.free += n; - jbf->offset += n; - - if (nxt_buf_is_file(jbf->buffer)) { - jbf->buffer->file_end += n; - } - - return NXT_OK; - } - - return NXT_ERROR; -} - - -static nxt_int_t -nxt_job_file_read_required(nxt_job_file_t *jbf) -{ - return NXT_OK; -} diff --git a/src/nxt_job_file.h b/src/nxt_job_file.h deleted file mode 100644 index 93c6393c..00000000 --- a/src/nxt_job_file.h +++ /dev/null @@ -1,74 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_JOB_FILE_H_INCLUDED_ -#define _NXT_JOB_FILE_H_INCLUDED_ - - -/* - * nxt_job_file_read() allows to open a file, to get its type, size, and - * modification time, to read or map file content to memory, and to close - * the file. It can be done as one operation for small file or as several - * operations for large file. On each operation completion ready_handler - * or error_handler completion handlers are called. Since they are job - * operations, they can be run by a thread pool. - * - * If a file is not opened then it is opened and its type, size, and - * modification time are got. Then file content starting from given offset - * is read or mapped in memory if there is a buffer supplied. The offset - * field is correspondingly updated. - * - * If there is no buffer but the read_ahead flag is set then the first - * byte is read to initiate read ahead operation. - * - * If the close flag is set then file descriptor is closed when the file - * is completely read. - * - * The complete flag is set by nxt_job_file_read() when the file is - * completely read. - * - * The test_before_open flag allows to save syscalls in some case, for - * example, not to open and then not to close a directory. It calls - * nxt_file_info() to get file type, size, and modification time before - * opening the file. A custom read_required() callback combined with this - * flag can also omit opening and reading on some conditions. However, - * if the callback forces opening then additional nxt_file_info() is - * called after opening. The default read_required() callback always - * forces opening and reading. - */ - - -typedef struct nxt_job_file_s nxt_job_file_t; - -struct nxt_job_file_s { - nxt_job_t job; - - nxt_file_t file; - - nxt_off_t offset; - nxt_buf_t *buffer; - - nxt_work_handler_t ready_handler; - nxt_work_handler_t error_handler; - - nxt_int_t (*read_required)(nxt_job_file_t *jbf); - - uint16_t directory_end; - - uint16_t close_before_open:1; - uint16_t test_before_open:1; - uint16_t read_ahead:1; - uint16_t close:1; - uint16_t complete:1; -}; - - -NXT_EXPORT nxt_job_file_t *nxt_job_file_create(nxt_mp_t *mp); -NXT_EXPORT void nxt_job_file_init(nxt_job_file_t *jbf); -NXT_EXPORT void nxt_job_file_read(nxt_task_t *task, nxt_job_t *job); - - -#endif /* _NXT_JOB_FILE_H_INCLUDED_ */ diff --git a/src/nxt_job_file_cache.c b/src/nxt_job_file_cache.c deleted file mode 100644 index 680d0665..00000000 --- a/src/nxt_job_file_cache.c +++ /dev/null @@ -1,42 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - - -#include - - -typedef struct { - nxt_cache_node_t node; - nxt_file_t file; -} nxt_file_cache_t; - - -void -nxt_job_file_cache_read(nxt_cache_t *cache, nxt_job_file_t *jbf) -{ - nxt_file_cache_node_t *node; - - node = nxt_cache_find(cache); - - if (node != NULL) { - - if (node->fd != -1) { - nxt_job_return(&jbf->job, jbf->ready_handler); - return; - } - - if (node->error != 0) { - nxt_job_return(&jbf->job, jbf->error_handler); - return; - } - - if (node->accessed + 60 > nxt_thread_time()) { - jbf->job.thread_pool = NULL; - } - } - - nxt_job_file_read(jbf); -} diff --git a/src/nxt_main.h b/src/nxt_main.h index b0cdc2d3..75e1724e 100644 --- a/src/nxt_main.h +++ b/src/nxt_main.h @@ -145,13 +145,9 @@ typedef void (*nxt_event_conn_handler_t)(nxt_thread_t *thr, nxt_conn_t *c); #include #include -#include -#include #include -#include - #include #include #include diff --git a/src/nxt_mem_pool_cleanup.c b/src/nxt_mem_pool_cleanup.c deleted file mode 100644 index ceafc9c8..00000000 --- a/src/nxt_mem_pool_cleanup.c +++ /dev/null @@ -1,39 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -static void nxt_mem_pool_file_cleanup_handler(nxt_task_t *task, void *data); - - -nxt_mem_pool_cleanup_t * -nxt_mem_pool_file_cleanup(nxt_mem_pool_t *mp, nxt_file_t *file) -{ - nxt_mem_pool_cleanup_t *mpcl; - - mpcl = nxt_mem_pool_cleanup(mp, 0); - - if (nxt_fast_path(mpcl != NULL)) { - mpcl->handler = nxt_mem_pool_file_cleanup_handler; - mpcl->data = file; - } - - return mpcl; -} - - -static void -nxt_mem_pool_file_cleanup_handler(nxt_task_t *task, void *data) -{ - nxt_file_t *file; - - file = data; - - if (file->fd != NXT_FILE_INVALID) { - nxt_file_close(task, file); - } -} diff --git a/src/nxt_mem_pool_cleanup.h b/src/nxt_mem_pool_cleanup.h deleted file mode 100644 index f84395d0..00000000 --- a/src/nxt_mem_pool_cleanup.h +++ /dev/null @@ -1,15 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_MEM_POOL_CLEANUP_H_INCLUDED_ -#define _NXT_MEM_POOL_CLEANUP_H_INCLUDED_ - - -NXT_EXPORT nxt_mem_pool_cleanup_t *nxt_mem_pool_file_cleanup(nxt_mem_pool_t *mp, - nxt_file_t *file); - - -#endif /* _NXT_MEM_POOL_CLEANUP_H_INCLUDED_ */ diff --git a/src/nxt_stream_module.c b/src/nxt_stream_module.c deleted file mode 100644 index 25aaec57..00000000 --- a/src/nxt_stream_module.c +++ /dev/null @@ -1,131 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include -#include - - -static void nxt_stream_connection_peer(nxt_task_t *task, - nxt_upstream_peer_t *up); -static void nxt_stream_connection_close(nxt_task_t *task, void *obj, - void *data); - - -void -nxt_stream_connection_init(nxt_task_t *task, void *obj, void *data) -{ - nxt_conn_t *c; - nxt_runtime_t *rt; - nxt_upstream_peer_t *up; - - c = obj; - - nxt_debug(task, "stream connection init"); - - up = nxt_mp_zget(c->mem_pool, sizeof(nxt_upstream_peer_t)); - if (nxt_slow_path(up == NULL)) { - goto fail; - } - - up->data = c; - - rt = task->thread->runtime; - - if (rt->upstream.length != 0) { - up->addr = rt->upstream; - - } else { - nxt_str_set(&up->addr, "127.0.0.1:8080"); - } - - up->ready_handler = nxt_stream_connection_peer; - up->mem_pool = c->mem_pool; - - nxt_upstream_round_robin_peer(task, up); - return; - -fail: - - /* TODO: close connection */ - return; -} - - -static void -nxt_stream_connection_peer(nxt_task_t *task, nxt_upstream_peer_t *up) -{ - nxt_conn_t *c; - nxt_conn_proxy_t *p; - - c = up->data; - - up->sockaddr->type = SOCK_STREAM; - - nxt_log_debug(c->socket.log, "stream connection peer %*s", - (size_t) up->sockaddr->length, - nxt_sockaddr_start(up->sockaddr)); - - p = nxt_conn_proxy_create(c); - if (nxt_slow_path(p == NULL)) { - goto fail; - } - - p->client->socket.data = p; - p->peer->socket.data = p; - - p->client_buffer_size = 1024; - p->peer_buffer_size = 4096; - //p->client_wait_timeout = 9000; - p->connect_timeout = 7000; - p->reconnect_timeout = 500; - //p->peer_wait_timeout = 5000; - p->client_write_timeout = 3000; - p->peer_write_timeout = 3000; - p->completion_handler = nxt_stream_connection_close; - //p->retries = 10; - p->peer->remote = up->sockaddr; - - if (0) { - nxt_event_engine_t *engine; - nxt_event_write_rate_t *rate; - - rate = nxt_mp_get(c->mem_pool, sizeof(nxt_event_write_rate_t)); - - if (nxt_slow_path(rate == NULL)) { - goto fail; - } - - c->rate = rate; - - rate->limit = 1024; - rate->limit_after = 0; - rate->average = rate->limit; - - engine = nxt_thread_event_engine(); - rate->last = engine->timers.now; - } - - nxt_conn_proxy(task, p); - return; - -fail: - - /* TODO: close connection */ - return; -} - - -static void -nxt_stream_connection_close(nxt_task_t *task, void *obj, void *data) -{ - nxt_event_conn_proxy_t *p; - - p = obj; - - nxt_log_debug(p->client->socket.log, "stream connection close"); - - nxt_mp_destroy(p->client->mem_pool); -} diff --git a/src/nxt_stream_source.c b/src/nxt_stream_source.c deleted file mode 100644 index 66ec1640..00000000 --- a/src/nxt_stream_source.c +++ /dev/null @@ -1,480 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -static void nxt_stream_source_connected(nxt_task_t *task, void *obj, - void *data); -static void nxt_stream_source_write_ready(nxt_task_t *task, void *obj, - void *data); -static void nxt_stream_source_read_ready(nxt_task_t *task, void *obj, - void *data); -static nxt_buf_t *nxt_stream_source_process_buffers(nxt_stream_source_t *stream, - nxt_event_conn_t *c); -static void nxt_stream_source_buf_completion(nxt_task_t *task, void *obj, - void *data); -static void nxt_stream_source_read_done(nxt_task_t *task, void *obj, - void *data); -static void nxt_stream_source_refused(nxt_task_t *task, void *obj, void *data); -static void nxt_stream_source_closed(nxt_task_t *task, void *obj, void *data); -static void nxt_stream_source_error(nxt_task_t *task, void *obj, void *data); -static void nxt_stream_source_close(nxt_task_t *task, - nxt_stream_source_t *stream); - - -static const nxt_event_conn_state_t nxt_stream_source_connect_state; -static const nxt_event_conn_state_t nxt_stream_source_request_write_state; -static const nxt_event_conn_state_t nxt_stream_source_response_ready_state; -static const nxt_event_conn_state_t nxt_stream_source_response_read_state; - - -void -nxt_stream_source_connect(nxt_task_t *task, nxt_stream_source_t *stream) -{ - nxt_thread_t *thr; - nxt_event_conn_t *c; - nxt_upstream_source_t *us; - - thr = nxt_thread(); - - us = stream->upstream; - - if (nxt_slow_path(!nxt_buf_pool_obtainable(&us->buffers))) { - nxt_log(task, NXT_LOG_ERR, - "%d buffers %uDK each are not enough to read upstream response", - us->buffers.max, us->buffers.size / 1024); - goto fail; - } - - c = nxt_event_conn_create(us->buffers.mem_pool, thr->log); - if (nxt_slow_path(c == NULL)) { - goto fail; - } - - stream->conn = c; - c->socket.data = stream; - - nxt_conn_work_queue_set(c, us->work_queue); - - c->remote = us->peer->sockaddr; - c->write_state = &nxt_stream_source_connect_state; - - nxt_event_conn_connect(task, c); - return; - -fail: - - stream->error_handler(task, stream); -} - - -static const nxt_event_conn_state_t nxt_stream_source_connect_state - nxt_aligned(64) = -{ - NXT_EVENT_NO_BUF_PROCESS, - NXT_EVENT_TIMER_AUTORESET, - - nxt_stream_source_connected, - nxt_stream_source_refused, - nxt_stream_source_error, - - NULL, /* timeout */ - NULL, /* timeout value */ - 0, /* connect_timeout */ -}; - - -static void -nxt_stream_source_connected(nxt_task_t *task, void *obj, void *data) -{ - nxt_event_conn_t *c; - nxt_stream_source_t *stream; - - c = obj; - stream = data; - - nxt_debug(task, "stream source connected fd:%d", c->socket.fd); - - c->read_state = &nxt_stream_source_response_ready_state; - c->write = stream->out; - c->write_state = &nxt_stream_source_request_write_state; - - if (task->thread->engine->batch != 0) { - nxt_event_conn_write(task, c); - - } else { - stream->read_queued = 1; - nxt_thread_work_queue_add(task->thread, - &task->thread->engine->read_work_queue, - c->io->read, task, c, stream); - - c->io->write(task, c, stream); - } -} - - -static const nxt_event_conn_state_t nxt_stream_source_request_write_state - nxt_aligned(64) = -{ - NXT_EVENT_NO_BUF_PROCESS, - NXT_EVENT_TIMER_AUTORESET, - - nxt_stream_source_write_ready, - NULL, - nxt_stream_source_error, - - NULL, /* timeout */ - NULL, /* timeout value */ - 0, /* connect_timeout */ -}; - - -static const nxt_event_conn_state_t nxt_stream_source_response_ready_state - nxt_aligned(64) = -{ - NXT_EVENT_NO_BUF_PROCESS, - NXT_EVENT_TIMER_AUTORESET, - - nxt_stream_source_read_ready, - nxt_stream_source_closed, - nxt_stream_source_error, - - NULL, /* timeout */ - NULL, /* timeout value */ - 0, /* connect_timeout */ -}; - - -static void -nxt_stream_source_write_ready(nxt_task_t *task, void *obj, void *data) -{ - nxt_event_conn_t *c; - - c = obj; - - nxt_debug(task, "stream source write ready fd:%d", c->socket.fd); - - nxt_conn_read(task, c); -} - - -static void -nxt_stream_source_read_ready(nxt_task_t *task, void *obj, void *data) -{ - nxt_int_t ret; - nxt_buf_t *b; - nxt_buf_pool_t *buffers; - nxt_event_conn_t *c; - nxt_stream_source_t *stream; - - c = obj; - stream = data; - stream->read_queued = 0; - - nxt_debug(task, "stream source read ready fd:%d", c->socket.fd); - - if (c->read == NULL) { - - buffers = &stream->upstream->buffers; - - ret = nxt_buf_pool_mem_alloc(buffers, 0); - - if (nxt_slow_path(ret != NXT_OK)) { - - if (nxt_slow_path(ret == NXT_ERROR)) { - goto fail; - } - - /* ret == NXT_AGAIN */ - - nxt_debug(task, "stream source flush"); - - b = nxt_buf_sync_alloc(buffers->mem_pool, NXT_BUF_SYNC_NOBUF); - - if (nxt_slow_path(b == NULL)) { - goto fail; - } - - nxt_event_fd_block_read(task->thread->engine, &c->socket); - - nxt_source_filter(task->thread, c->write_work_queue, task, - stream->next, b); - return; - } - - c->read = buffers->current; - buffers->current = NULL; - } - - c->read_state = &nxt_stream_source_response_read_state; - - nxt_conn_read(task, c); - return; - -fail: - - nxt_stream_source_close(task, stream); -} - - -static const nxt_event_conn_state_t nxt_stream_source_response_read_state - nxt_aligned(64) = -{ - NXT_EVENT_NO_BUF_PROCESS, - NXT_EVENT_TIMER_AUTORESET, - - nxt_stream_source_read_done, - nxt_stream_source_closed, - nxt_stream_source_error, - - NULL, /* timeout */ - NULL, /* timeout value */ - 0, /* connect_timeout */ -}; - - -static void -nxt_stream_source_read_done(nxt_task_t *task, void *obj, void *data) -{ - nxt_buf_t *b; - nxt_bool_t batch; - nxt_event_conn_t *c; - nxt_stream_source_t *stream; - - c = obj; - stream = data; - - nxt_debug(task, "stream source read done fd:%d", c->socket.fd); - - if (c->read != NULL) { - b = nxt_stream_source_process_buffers(stream, c); - - if (nxt_slow_path(b == NULL)) { - nxt_stream_source_close(task, stream); - return; - } - - batch = (task->thread->engine->batch != 0); - - if (batch) { - nxt_thread_work_queue_add(task->thread, - stream->upstream->work_queue, - nxt_source_filter_handler, - task, stream->next, b); - } - - if (!stream->read_queued) { - stream->read_queued = 1; - nxt_thread_work_queue_add(task->thread, - stream->upstream->work_queue, - nxt_stream_source_read_ready, - task, c, stream); - } - - if (!batch) { - stream->next->filter(task, stream->next->context, b); - } - } -} - - -static nxt_buf_t * -nxt_stream_source_process_buffers(nxt_stream_source_t *stream, - nxt_event_conn_t *c) -{ - size_t size, nbytes; - nxt_buf_t *b, *in, *head, **prev; - - nbytes = c->nbytes; - prev = &head; - - do { - b = nxt_buf_mem_alloc(stream->upstream->buffers.mem_pool, 0, 0); - - if (nxt_slow_path(b == NULL)) { - return NULL; - } - - *prev = b; - - b->data = stream; - b->completion_handler = nxt_stream_source_buf_completion; - - in = c->read; - in->retain++; - b->parent = in; - - b->mem.pos = in->mem.free; - b->mem.start = in->mem.free; - - size = nxt_buf_mem_free_size(&in->mem); - - if (nbytes < size) { - in->mem.free += nbytes; - - b->mem.free = in->mem.free; - b->mem.end = in->mem.free; - - break; - } - - in->mem.free = in->mem.end; - - b->mem.free = in->mem.free; - b->mem.end = in->mem.free; - nbytes -= size; - - prev = &b->next; - c->read = in->next; - in->next = NULL; - - } while (c->read != NULL); - - return head; -} - - -static void -nxt_stream_source_buf_completion(nxt_task_t *task, void *obj, void *data) -{ - size_t size; - nxt_buf_t *b, *parent; - nxt_stream_source_t *stream; - - b = obj; - parent = data; - -#if 0 - nxt_debug(thr->log, - "stream source buf completion: %p parent:%p retain:%uD", - b, parent, parent->retain); -#endif - - stream = b->data; - - /* A parent is a buffer where stream reads data. */ - - parent->mem.pos = b->mem.pos; - parent->retain--; - - if (parent->retain == 0 && !stream->conn->socket.closed) { - size = nxt_buf_mem_size(&parent->mem); - - parent->mem.pos = parent->mem.start; - parent->mem.free = parent->mem.start; - - /* - * A buffer's original size can be changed by filters - * so reuse the buffer only if it is still large enough. - */ - if (size >= 256 || size >= stream->upstream->buffers.size) { - - if (stream->conn->read != parent) { - nxt_buf_chain_add(&stream->conn->read, parent); - } - - if (!stream->read_queued) { - stream->read_queued = 1; - nxt_thread_work_queue_add(task->thread, - stream->upstream->work_queue, - nxt_stream_source_read_ready, - task, stream->conn, - stream->conn->socket.data); - } - } - } - - nxt_buf_free(stream->upstream->buffers.mem_pool, b); -} - - -static void -nxt_stream_source_refused(nxt_task_t *task, void *obj, void *data) -{ - nxt_stream_source_t *stream; - - stream = data; - -#if (NXT_DEBUG) - { - nxt_event_conn_t *c; - - c = obj; - - nxt_debug(task, "stream source refused fd:%d", c->socket.fd); - } -#endif - - nxt_stream_source_close(task, stream); -} - - -static void -nxt_stream_source_closed(nxt_task_t *task, void *obj, void *data) -{ - nxt_buf_t *b; - nxt_event_conn_t *c; - nxt_stream_source_t *stream; - - c = obj; - stream = data; - - nxt_debug(task, "stream source closed fd:%d", c->socket.fd); - - nxt_conn_close(task, c); - - b = nxt_buf_sync_alloc(stream->upstream->buffers.mem_pool, - NXT_BUF_SYNC_LAST); - - if (nxt_slow_path(b == NULL)) { - stream->error_handler(task, stream); - return; - } - - nxt_source_filter(task->thread, c->write_work_queue, task, stream->next, b); -} - - -static void -nxt_stream_source_error(nxt_task_t *task, void *obj, void *data) -{ - nxt_stream_source_t *stream; - - stream = data; - -#if (NXT_DEBUG) - { - nxt_event_fd_t *ev; - - ev = obj; - - nxt_debug(task, "stream source error fd:%d", ev->fd); - } -#endif - - nxt_stream_source_close(task, stream); -} - - -static void -nxt_stream_source_close(nxt_task_t *task, nxt_stream_source_t *stream) -{ - nxt_conn_close(task, stream->conn); - - stream->error_handler(task, stream); -} - - -void -nxt_source_filter_handler(nxt_task_t *task, void *obj, void *data) -{ - nxt_source_hook_t *next; - - next = obj; - - next->filter(task, next->context, data); -} diff --git a/src/nxt_stream_source.h b/src/nxt_stream_source.h deleted file mode 100644 index 2d57073f..00000000 --- a/src/nxt_stream_source.h +++ /dev/null @@ -1,32 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_STREAM_SOURCE_H_INCLUDED_ -#define _NXT_STREAM_SOURCE_H_INCLUDED_ - - -typedef struct nxt_stream_source_s nxt_stream_source_t; - -typedef void (*nxt_stream_source_handler_t)(nxt_task_t *task, - nxt_stream_source_t *s); - -struct nxt_stream_source_s { - nxt_conn_t *conn; - nxt_source_hook_t *next; - nxt_upstream_source_t *upstream; - - nxt_buf_t *out; - - uint32_t read_queued; /* 1 bit */ - - nxt_stream_source_handler_t error_handler; -}; - - -void nxt_stream_source_connect(nxt_task_t *task, nxt_stream_source_t *stream); - - -#endif /* _NXT_STREAM_SOURCE_H_INCLUDED_ */ diff --git a/src/nxt_upstream_source.c b/src/nxt_upstream_source.c deleted file mode 100644 index ee3fc21e..00000000 --- a/src/nxt_upstream_source.c +++ /dev/null @@ -1,71 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -static nxt_int_t nxt_upstream_header_hash_test(nxt_lvlhsh_query_t *lhq, - void *data); - - -const nxt_lvlhsh_proto_t nxt_upstream_header_hash_proto nxt_aligned(64) = { - NXT_LVLHSH_DEFAULT, - 0, - nxt_upstream_header_hash_test, - nxt_mem_lvlhsh_alloc, - nxt_mem_lvlhsh_free, -}; - - -nxt_int_t -nxt_upstream_header_hash_add(nxt_mp_t *mp, nxt_lvlhsh_t *lh, - const nxt_upstream_name_value_t *unv, nxt_uint_t n) -{ - nxt_lvlhsh_query_t lhq; - - while (n != 0) { - lhq.key_hash = nxt_djb_hash(unv->name, unv->len); - lhq.replace = 1; - lhq.key.len = unv->len; - lhq.key.data = (u_char *) unv->name; - lhq.value = (void *) unv; - lhq.proto = &nxt_upstream_header_hash_proto; - lhq.pool = mp; - - if (nxt_lvlhsh_insert(lh, &lhq) != NXT_OK) { - return NXT_ERROR; - } - - unv++; - n--; - } - - return NXT_OK; -} - - -static nxt_int_t -nxt_upstream_header_hash_test(nxt_lvlhsh_query_t *lhq, void *data) -{ - nxt_upstream_name_value_t *unv; - - unv = data; - - if (lhq->key.len == unv->len - && nxt_memcasecmp(lhq->key.data, unv->name, unv->len) == 0) - { - return NXT_OK; - } - - return NXT_DECLINED; -} - - -nxt_int_t -nxt_upstream_name_value_ignore(nxt_upstream_source_t *us, nxt_name_value_t *nv) -{ - return NXT_OK; -} diff --git a/src/nxt_upstream_source.h b/src/nxt_upstream_source.h deleted file mode 100644 index 143b8d0c..00000000 --- a/src/nxt_upstream_source.h +++ /dev/null @@ -1,83 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_UPSTREAM_SOURCE_H_INCLUDED_ -#define _NXT_UPSTREAM_SOURCE_H_INCLUDED_ - - -typedef struct { - uint32_t hash; - - unsigned value_len:23; - unsigned skip:1; - unsigned name_len:8; - - u_char *value_start; - u_char *name_start; -} nxt_name_value_t; - - -typedef struct { - nxt_list_t *list; - nxt_lvlhsh_t hash; - - uint16_t status; /* 16 bits */ - - nxt_off_t content_length; -} nxt_upstream_header_in_t; - - -typedef nxt_int_t (*nxt_upstream_name_value_handler_t)( - nxt_upstream_source_t *us, nxt_name_value_t *nv); - - -typedef struct { - nxt_upstream_name_value_handler_t handler; - - uint8_t len; - /* - * A name is inlined to test it with one memory access. - * The struct size is aligned to 32 bytes. - */ -#if (NXT_64BIT) - u_char name[23]; -#else - u_char name[27]; -#endif -} nxt_upstream_name_value_t; - - -struct nxt_upstream_source_s { - nxt_upstream_peer_t *peer; - - const nxt_upstream_state_t *state; - - void *protocol_source; - void *data; - nxt_work_queue_t *work_queue; - - nxt_buf_pool_t buffers; - - nxt_lvlhsh_t header_hash; - nxt_stream_source_t *stream; -}; - - -#define NXT_UPSTREAM_NAME_VALUE_MIN_SIZE \ - offsetof(nxt_http_upstream_header_t, name) - -#define nxt_upstream_name_value(s) nxt_length(s), s - - -NXT_EXPORT nxt_int_t nxt_upstream_header_hash_add(nxt_mp_t *mp, - nxt_lvlhsh_t *lh, const nxt_upstream_name_value_t *unv, nxt_uint_t n); -NXT_EXPORT nxt_int_t nxt_upstream_name_value_ignore(nxt_upstream_source_t *us, - nxt_name_value_t *nv); - -NXT_EXPORT extern const nxt_lvlhsh_proto_t nxt_upstream_header_hash_proto; - - -#endif /* _NXT_UPSTREAM_SOURCE_H_INCLUDED_ */ -- cgit From 375556f9aa76c1b9ff77d08f75451dfffb1e082a Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Fri, 24 Mar 2023 14:23:06 +0000 Subject: Don't conflate the error variable in nxt_kqueue_poll(). In nxt_kqueue_poll() error is declared as a nxt_bool_t aka unsigned int (on x86-64 anyway). It is used both as a boolean and as the return storage for a bitwise AND operation. This has potential to go awry. If nxt_bool_t was changed to be a u8 then we would have the following issue gcc12 -c -pipe -fPIC -fvisibility=hidden -O -W -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wmissing-prototypes -Werror -g -O2 -I src -I build -I/usr/local/include -o build/src/nxt_kqueue_engine.o -MMD -MF build/src/nxt_kqueue_engine.dep -MT build/src/nxt_kqueue_engine.o src/nxt_kqueue_engine.c src/nxt_kqueue_engine.c: In function 'nxt_kqueue_poll': src/nxt_kqueue_engine.c:728:17: error: overflow in conversion from 'int' to 'nxt_bool_t' {aka 'unsigned char'} changes value from '(int)kev->flags & 16384' to '0' [-Werror=overflow] 728 | error = (kev->flags & EV_ERROR); | ^ cc1: all warnings being treated as errors EV_ERROR has the value 16384, after the AND operation error holds 16384, however this overflows and wraps around (64 times) exactly to 0. With nxt_bool_t defined as a u32, we would have a similar issue if EV_ERROR ever became UINT_MAX + 1 (or a multiple thereof)... Rather than conflating the use of error, keep error as a boolean (it is used further down the function) but do the AND operation inside the if (). Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_kqueue_engine.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/nxt_kqueue_engine.c b/src/nxt_kqueue_engine.c index ecc3251e..a7a5a29e 100644 --- a/src/nxt_kqueue_engine.c +++ b/src/nxt_kqueue_engine.c @@ -716,6 +716,8 @@ nxt_kqueue_poll(nxt_event_engine_t *engine, nxt_msec_t timeout) for (i = 0; i < nevents; i++) { + error = 0; + kev = &engine->u.kqueue.events[i]; nxt_debug(&engine->task, @@ -725,12 +727,11 @@ nxt_kqueue_poll(nxt_event_engine_t *engine, nxt_msec_t timeout) kev->ident, kev->filter, kev->flags, kev->fflags, kev->data, kev->udata); - error = (kev->flags & EV_ERROR); - - if (nxt_slow_path(error)) { + if (nxt_slow_path(kev->flags & EV_ERROR)) { nxt_alert(&engine->task, "kevent(%d) error %E on ident:%d filter:%d", engine->u.kqueue.fd, kev->data, kev->ident, kev->filter); + error = 1; } task = &engine->task; -- cgit From dcdc8e7466dcae2444166134114881a30c124bf2 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Thu, 23 Mar 2023 21:01:40 +0000 Subject: Remove an erroneous semi-colon. Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_conf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/nxt_conf.c b/src/nxt_conf.c index d04aa45c..664b5468 100644 --- a/src/nxt_conf.c +++ b/src/nxt_conf.c @@ -46,7 +46,7 @@ typedef struct nxt_conf_object_s nxt_conf_object_t; struct nxt_conf_value_s { union { uint8_t boolean; /* 1 bit. */ - u_char number[NXT_CONF_MAX_NUMBER_LEN + 1];; + u_char number[NXT_CONF_MAX_NUMBER_LEN + 1]; struct { u_char start[NXT_CONF_MAX_SHORT_STRING]; -- cgit From 1fd6eb626baeee94a65b199cd848019e0e6b81b3 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Tue, 21 Mar 2023 22:46:42 +0000 Subject: Decouple "Unit" from NXT_SERVER. Split out the "Unit" name from the NXT_SERVER #define into its own NXT_NAME #define, then make NXT_SERVER a combination of that and NXT_VERSION. This is required for a subsequent commit where we may want the server name on its own. Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_main.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/nxt_main.h b/src/nxt_main.h index 75e1724e..a7e0c283 100644 --- a/src/nxt_main.h +++ b/src/nxt_main.h @@ -11,7 +11,8 @@ #include #include -#define NXT_SERVER "Unit/" NXT_VERSION +#define NXT_NAME "Unit" +#define NXT_SERVER NXT_NAME "/" NXT_VERSION typedef struct nxt_port_s nxt_port_t; typedef struct nxt_task_s nxt_task_t; -- cgit From 1a485fed6a8353ecc09e6c0f050e44c0a2d30419 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Sat, 18 Mar 2023 16:32:59 +0000 Subject: Allow to remove the version string in HTTP responses. Normally Unit responds to HTTP requests by including a header like Server: Unit/1.30.0 however it can sometimes be beneficial to withhold the version information and in this case just respond with Server: Unit This patch adds a new "settings.http" boolean option called server_version, which defaults to true, in which case the full version information is sent. However this can be set to false, e.g "settings": { "http": { "server_version": false } }, in which case Unit responds without the version information as the latter example above shows. Link: Closes: Reviewed-by: Alejandro Colomar Signed-off-by: Andrew Clayton --- src/nxt_conf_validation.c | 3 +++ src/nxt_http_request.c | 12 +++++++++--- src/nxt_router.c | 8 ++++++++ src/nxt_router.h | 2 ++ 4 files changed, 22 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 6d798a81..9059b2b3 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -347,6 +347,9 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { }, { .name = nxt_string("log_route"), .type = NXT_CONF_VLDT_BOOLEAN, + }, { + .name = nxt_string("server_version"), + .type = NXT_CONF_VLDT_BOOLEAN, }, NXT_CONF_VLDT_END diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index e78975aa..e53b1ec8 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -622,8 +622,9 @@ void nxt_http_request_header_send(nxt_task_t *task, nxt_http_request_t *r, nxt_work_handler_t body_handler, void *data) { - u_char *p, *end; - nxt_http_field_t *server, *date, *content_length; + u_char *p, *end, *server_string; + nxt_http_field_t *server, *date, *content_length; + nxt_socket_conf_t *skcf; /* * TODO: "Server", "Date", and "Content-Length" processing should be moved @@ -635,7 +636,12 @@ nxt_http_request_header_send(nxt_task_t *task, nxt_http_request_t *r, goto fail; } - nxt_http_field_set(server, "Server", NXT_SERVER); + skcf = r->conf->socket_conf; + server_string = (u_char *) (skcf->server_version ? NXT_SERVER : NXT_NAME); + + nxt_http_field_name_set(server, "Server"); + server->value = server_string; + server->value_length = nxt_strlen(server_string); if (r->resp.date == NULL) { date = nxt_list_zero_add(r->resp.fields); diff --git a/src/nxt_router.c b/src/nxt_router.c index 992cc039..c4e29e3a 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1519,6 +1519,12 @@ static nxt_conf_map_t nxt_router_http_conf[] = { NXT_CONF_MAP_INT8, offsetof(nxt_socket_conf_t, log_route), }, + + { + nxt_string("server_version"), + NXT_CONF_MAP_INT8, + offsetof(nxt_socket_conf_t, server_version), + }, }; @@ -1927,6 +1933,8 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, skcf->proxy_send_timeout = 30 * 1000; skcf->proxy_read_timeout = 30 * 1000; + skcf->server_version = 1; + skcf->websocket_conf.max_frame_size = 1024 * 1024; skcf->websocket_conf.read_timeout = 60 * 1000; skcf->websocket_conf.keepalive_interval = 30 * 1000; diff --git a/src/nxt_router.h b/src/nxt_router.h index 64095b69..6f0ba5ad 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -201,6 +201,8 @@ typedef struct { uint8_t discard_unsafe_fields; /* 1 bit */ + uint8_t server_version; /* 1 bit */ + nxt_http_forward_t *forwarded; nxt_http_forward_t *client_ip; -- cgit From 14d6d97bacf9b06ba340ebd4211b2f1b6ad417dd Mon Sep 17 00:00:00 2001 From: Zhidao HONG Date: Thu, 20 Apr 2023 23:20:41 +0800 Subject: HTTP: added basic URI rewrite. This commit introduced the basic URI rewrite. It allows users to change request URI. Note the "rewrite" option ignores the contained query if any and the query from the request is preserverd. An example: "routes": [ { "match": { "uri": "/v1/test" }, "action": { "return": 200 } }, { "action": { "rewrite": "/v1$uri", "pass": "routes" } } ] Reviewed-by: Alejandro Colomar --- src/nxt_conf_validation.c | 18 ++++++-- src/nxt_http.h | 7 ++++ src/nxt_http_parse.c | 4 +- src/nxt_http_parse.h | 1 + src/nxt_http_request.c | 9 ++++ src/nxt_http_rewrite.c | 104 ++++++++++++++++++++++++++++++++++++++++++++++ src/nxt_http_route.c | 20 +++++++-- src/nxt_tstr.c | 7 ++++ src/nxt_tstr.h | 1 + 9 files changed, 160 insertions(+), 11 deletions(-) create mode 100644 src/nxt_http_rewrite.c (limited to 'src') diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 9059b2b3..c9aac790 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -669,6 +669,16 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_match_members[] = { }; +static nxt_conf_vldt_object_t nxt_conf_vldt_action_common_members[] = { + { + .name = nxt_string("rewrite"), + .type = NXT_CONF_VLDT_STRING, + }, + + NXT_CONF_VLDT_END +}; + + static nxt_conf_vldt_object_t nxt_conf_vldt_pass_action_members[] = { { .name = nxt_string("pass"), @@ -677,7 +687,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_pass_action_members[] = { .flags = NXT_CONF_VLDT_TSTR, }, - NXT_CONF_VLDT_END + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_action_common_members) }; @@ -692,7 +702,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_return_action_members[] = { .flags = NXT_CONF_VLDT_TSTR, }, - NXT_CONF_VLDT_END + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_action_common_members) }; @@ -736,7 +746,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_share_action_members[] = { #endif }, - NXT_CONF_VLDT_END + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_action_common_members) }; @@ -747,7 +757,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_proxy_action_members[] = { .validator = nxt_conf_vldt_proxy, }, - NXT_CONF_VLDT_END + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_action_common_members) }; diff --git a/src/nxt_http.h b/src/nxt_http.h index d7bbaf02..08e1fcbe 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -226,6 +226,7 @@ typedef struct nxt_http_route_addr_rule_s nxt_http_route_addr_rule_t; typedef struct { + nxt_conf_value_t *rewrite; nxt_conf_value_t *pass; nxt_conf_value_t *ret; nxt_conf_value_t *location; @@ -253,6 +254,7 @@ struct nxt_http_action_s { nxt_str_t *pass; } u; + nxt_tstr_t *rewrite; nxt_http_action_t *fallback; }; @@ -378,6 +380,11 @@ nxt_int_t nxt_upstreams_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_int_t nxt_upstreams_joint_create(nxt_router_temp_conf_t *tmcf, nxt_upstream_t ***upstream_joint); +nxt_int_t nxt_http_rewrite_init(nxt_router_conf_t *rtcf, + nxt_http_action_t *action, nxt_http_action_conf_t *acf); +nxt_int_t nxt_http_rewrite(nxt_task_t *task, nxt_http_request_t *r, + nxt_http_action_t *action); + nxt_int_t nxt_http_return_init(nxt_router_conf_t *rtcf, nxt_http_action_t *action, nxt_http_action_conf_t *acf); diff --git a/src/nxt_http_parse.c b/src/nxt_http_parse.c index 439993df..50cbda2b 100644 --- a/src/nxt_http_parse.c +++ b/src/nxt_http_parse.c @@ -19,8 +19,6 @@ static u_char *nxt_http_lookup_field_end(u_char *p, const u_char *end); static nxt_int_t nxt_http_parse_field_end(nxt_http_request_parse_t *rp, u_char **pos, const u_char *end); -static nxt_int_t nxt_http_parse_complex_target(nxt_http_request_parse_t *rp); - static nxt_int_t nxt_http_field_hash_test(nxt_lvlhsh_query_t *lhq, void *data); static nxt_int_t nxt_http_field_hash_collision(nxt_lvlhsh_query_t *lhq, @@ -854,7 +852,7 @@ static const uint8_t nxt_http_normal[32] nxt_aligned(32) = { }; -static nxt_int_t +nxt_int_t nxt_http_parse_complex_target(nxt_http_request_parse_t *rp) { u_char *p, *u, c, ch, high, *args; diff --git a/src/nxt_http_parse.h b/src/nxt_http_parse.h index 88b10675..fa95e842 100644 --- a/src/nxt_http_parse.h +++ b/src/nxt_http_parse.h @@ -127,6 +127,7 @@ nxt_uint_t nxt_http_fields_hash_collisions(nxt_lvlhsh_t *hash, nxt_int_t nxt_http_fields_process(nxt_list_t *fields, nxt_lvlhsh_t *hash, void *ctx); +nxt_int_t nxt_http_parse_complex_target(nxt_http_request_parse_t *rp); nxt_buf_t *nxt_http_chunk_parse(nxt_task_t *task, nxt_http_chunk_parse_t *hcp, nxt_buf_t *in); diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index e53b1ec8..48f7dbe3 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -555,9 +555,18 @@ void nxt_http_request_action(nxt_task_t *task, nxt_http_request_t *r, nxt_http_action_t *action) { + nxt_int_t ret; + if (nxt_fast_path(action != NULL)) { do { + if (action->rewrite != NULL) { + ret = nxt_http_rewrite(task, r, action); + if (nxt_slow_path(ret != NXT_OK)) { + break; + } + } + action = action->handler(task, r, action); if (action == NULL) { diff --git a/src/nxt_http_rewrite.c b/src/nxt_http_rewrite.c new file mode 100644 index 00000000..b800a919 --- /dev/null +++ b/src/nxt_http_rewrite.c @@ -0,0 +1,104 @@ + +/* + * Copyright (C) Zhidao HONG + * Copyright (C) NGINX, Inc. + */ + +#include +#include + + +nxt_int_t +nxt_http_rewrite_init(nxt_router_conf_t *rtcf, nxt_http_action_t *action, + nxt_http_action_conf_t *acf) + { + nxt_str_t str; + + nxt_conf_get_string(acf->rewrite, &str); + + action->rewrite = nxt_tstr_compile(rtcf->tstr_state, &str, 0); + if (nxt_slow_path(action->rewrite == NULL)) { + return NXT_ERROR; + } + + return NXT_OK; +} + + +nxt_int_t +nxt_http_rewrite(nxt_task_t *task, nxt_http_request_t *r, + nxt_http_action_t *action) +{ + u_char *p; + nxt_int_t ret; + nxt_str_t str, encoded_path, target; + nxt_router_conf_t *rtcf; + nxt_http_request_parse_t rp; + + if (nxt_tstr_is_const(action->rewrite)) { + nxt_tstr_str(action->rewrite, &str); + + } else { + rtcf = r->conf->socket_conf->router_conf; + + ret = nxt_tstr_query_init(&r->tstr_query, rtcf->tstr_state, + &r->tstr_cache, r, r->mem_pool); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + nxt_tstr_query(task, r->tstr_query, action->rewrite, &str); + + if (nxt_slow_path(nxt_tstr_query_failed(r->tstr_query))) { + return NXT_ERROR; + } + } + + nxt_memzero(&rp, sizeof(nxt_http_request_parse_t)); + + rp.mem_pool = r->mem_pool; + + rp.target_start = str.start; + rp.target_end = str.start + str.length; + + ret = nxt_http_parse_complex_target(&rp); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + p = (rp.args.length > 0) ? rp.args.start - 1 : rp.target_end; + + encoded_path.start = rp.target_start; + encoded_path.length = p - encoded_path.start; + + if (r->args->length == 0) { + r->target = encoded_path; + + } else { + target.length = encoded_path.length + 1 + r->args->length; + + target.start = nxt_mp_alloc(r->mem_pool, target.length); + if (target.start == NULL) { + return NXT_ERROR; + } + + p = nxt_cpymem(target.start, encoded_path.start, encoded_path.length); + *p++ = '?'; + nxt_memcpy(p, r->args->start, r->args->length); + + r->target = target; + } + + r->path = nxt_mp_alloc(r->mem_pool, sizeof(nxt_str_t)); + if (nxt_slow_path(r->path == NULL)) { + return NXT_ERROR; + } + + *r->path = rp.path; + + if (nxt_slow_path(r->log_route)) { + nxt_log(task, NXT_LOG_NOTICE, "URI rewritten to \"%V\"", &r->target); + } + + return NXT_OK; +} diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index f439c957..0935dd4a 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -578,6 +578,11 @@ nxt_http_route_match_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, static nxt_conf_map_t nxt_http_route_action_conf[] = { + { + nxt_string("rewrite"), + NXT_CONF_MAP_PTR, + offsetof(nxt_http_action_conf_t, rewrite) + }, { nxt_string("pass"), NXT_CONF_MAP_PTR, @@ -659,6 +664,13 @@ nxt_http_action_init(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, rtcf = tmcf->router_conf; mp = rtcf->mem_pool; + if (acf.rewrite != NULL) { + ret = nxt_http_rewrite_init(rtcf, action, &acf); + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + } + if (acf.ret != NULL) { return nxt_http_return_init(rtcf, action, &acf); } @@ -1312,8 +1324,8 @@ nxt_http_pass_var(nxt_task_t *task, nxt_http_request_t *r, goto fail; } - action = nxt_mp_get(r->mem_pool, - sizeof(nxt_http_action_t) + sizeof(nxt_str_t)); + action = nxt_mp_zget(r->mem_pool, + sizeof(nxt_http_action_t) + sizeof(nxt_str_t)); if (nxt_slow_path(action == NULL)) { goto fail; } @@ -1496,7 +1508,7 @@ nxt_http_action_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, rtcf = tmcf->router_conf; mp = rtcf->mem_pool; - action = nxt_mp_alloc(mp, sizeof(nxt_http_action_t)); + action = nxt_mp_zalloc(mp, sizeof(nxt_http_action_t)); if (nxt_slow_path(action == NULL)) { return NULL; } @@ -1525,7 +1537,7 @@ nxt_http_pass_application(nxt_task_t *task, nxt_router_conf_t *rtcf, { nxt_http_action_t *action; - action = nxt_mp_alloc(rtcf->mem_pool, sizeof(nxt_http_action_t)); + action = nxt_mp_zalloc(rtcf->mem_pool, sizeof(nxt_http_action_t)); if (nxt_slow_path(action == NULL)) { return NULL; } diff --git a/src/nxt_tstr.c b/src/nxt_tstr.c index fda585b8..c439696e 100644 --- a/src/nxt_tstr.c +++ b/src/nxt_tstr.c @@ -296,6 +296,13 @@ nxt_tstr_query(nxt_task_t *task, nxt_tstr_query_t *query, nxt_tstr_t *tstr, } +nxt_bool_t +nxt_tstr_query_failed(nxt_tstr_query_t *query) +{ + return query->failed; +} + + void nxt_tstr_query_resolve(nxt_task_t *task, nxt_tstr_query_t *query, void *data, nxt_work_handler_t ready, nxt_work_handler_t error) diff --git a/src/nxt_tstr.h b/src/nxt_tstr.h index ce8e6f3a..afa7f56d 100644 --- a/src/nxt_tstr.h +++ b/src/nxt_tstr.h @@ -52,6 +52,7 @@ nxt_int_t nxt_tstr_query_init(nxt_tstr_query_t **query_p, nxt_mp_t *mp); void nxt_tstr_query(nxt_task_t *task, nxt_tstr_query_t *query, nxt_tstr_t *tstr, nxt_str_t *val); +nxt_bool_t nxt_tstr_query_failed(nxt_tstr_query_t *query); void nxt_tstr_query_resolve(nxt_task_t *task, nxt_tstr_query_t *query, void *data, nxt_work_handler_t ready, nxt_work_handler_t error); void nxt_tstr_query_handle(nxt_task_t *task, nxt_tstr_query_t *query, -- cgit From a3c3a29493798873ad04922bb2a7180b2ce267d5 Mon Sep 17 00:00:00 2001 From: Zhidao HONG Date: Mon, 8 May 2023 16:00:25 +0800 Subject: NJS: supported loadable modules. --- src/nxt_conf_validation.c | 74 ++++- src/nxt_controller.c | 353 +++++++++++++++++++++++ src/nxt_js.c | 248 ++++++++++++++-- src/nxt_js.h | 11 +- src/nxt_main_process.c | 7 + src/nxt_port.h | 6 + src/nxt_process.h | 3 + src/nxt_router.c | 158 ++++++++++- src/nxt_router.h | 4 + src/nxt_runtime.c | 17 ++ src/nxt_runtime.h | 1 + src/nxt_script.c | 709 ++++++++++++++++++++++++++++++++++++++++++++++ src/nxt_script.h | 37 +++ src/nxt_tstr.c | 5 +- 14 files changed, 1584 insertions(+), 49 deletions(-) create mode 100644 src/nxt_script.c create mode 100644 src/nxt_script.h (limited to 'src') diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index c9aac790..8c75a9fe 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -226,6 +227,13 @@ static nxt_int_t nxt_conf_vldt_cgroup_path(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); #endif +#if (NXT_HAVE_NJS) +static nxt_int_t nxt_conf_vldt_js_module(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_js_module_element(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value); +#endif + static nxt_conf_vldt_object_t nxt_conf_vldt_setting_members[]; static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[]; @@ -297,6 +305,12 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_setting_members[] = { .type = NXT_CONF_VLDT_OBJECT, .validator = nxt_conf_vldt_object, .u.members = nxt_conf_vldt_http_members, +#if (NXT_HAVE_NJS) + }, { + .name = nxt_string("js_module"), + .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_js_module, +#endif }, NXT_CONF_VLDT_END @@ -1306,35 +1320,26 @@ nxt_conf_validate(nxt_conf_validation_t *vldt) vldt->tstr_state = nxt_tstr_state_new(vldt->pool, 1); if (nxt_slow_path(vldt->tstr_state == NULL)) { - ret = NXT_ERROR; - goto fail; + return NXT_ERROR; } ret = nxt_conf_vldt_type(vldt, NULL, vldt->conf, NXT_CONF_VLDT_OBJECT); if (ret != NXT_OK) { - goto fail; + return ret; } ret = nxt_conf_vldt_object(vldt, vldt->conf, nxt_conf_vldt_root_members); if (ret != NXT_OK) { - goto fail; + return ret; } ret = nxt_tstr_state_done(vldt->tstr_state, error); if (ret != NXT_OK) { ret = nxt_conf_vldt_error(vldt, "%s", error); - goto fail; + return ret; } - nxt_tstr_state_release(vldt->tstr_state); - return NXT_OK; - -fail: - - nxt_tstr_state_release(vldt->tstr_state); - - return ret; } @@ -3241,6 +3246,49 @@ nxt_conf_vldt_server_weight(nxt_conf_validation_t *vldt, } +#if (NXT_HAVE_NJS) + +static nxt_int_t +nxt_conf_vldt_js_module(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, + void *data) +{ + if (nxt_conf_type(value) == NXT_CONF_ARRAY) { + return nxt_conf_vldt_array_iterator(vldt, value, + &nxt_conf_vldt_js_module_element); + } + + /* NXT_CONF_STRING */ + + return nxt_conf_vldt_js_module_element(vldt, value); +} + + +static nxt_int_t +nxt_conf_vldt_js_module_element(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value) +{ + nxt_str_t name; + nxt_conf_value_t *module; + + if (nxt_conf_type(value) != NXT_CONF_STRING) { + return nxt_conf_vldt_error(vldt, "The \"js_module\" array must " + "contain only string values."); + } + + nxt_conf_get_string(value, &name); + + module = nxt_script_info_get(&name); + if (module == NULL) { + return nxt_conf_vldt_error(vldt, "JS module \"%V\" is not found.", + &name); + } + + return NXT_OK; +} + +#endif + + typedef struct { nxt_str_t path; nxt_str_t format; diff --git a/src/nxt_controller.c b/src/nxt_controller.c index b5e0d831..4e2e3749 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -11,6 +11,7 @@ #include #include #include +#include typedef struct { @@ -101,6 +102,15 @@ static nxt_bool_t nxt_controller_cert_in_use(nxt_str_t *name); static void nxt_controller_cert_cleanup(nxt_task_t *task, void *obj, void *data); #endif +#if (NXT_HAVE_NJS) +static void nxt_controller_process_script(nxt_task_t *task, + nxt_controller_request_t *req, nxt_str_t *path); +static void nxt_controller_process_script_save(nxt_task_t *task, + nxt_port_recv_msg_t *msg, void *data); +static nxt_bool_t nxt_controller_script_in_use(nxt_str_t *name); +static void nxt_controller_script_cleanup(nxt_task_t *task, void *obj, + void *data); +#endif static void nxt_controller_process_control(nxt_task_t *task, nxt_controller_request_t *req, nxt_str_t *path); static void nxt_controller_app_restart_handler(nxt_task_t *task, @@ -213,6 +223,13 @@ nxt_controller_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp) nxt_mp_cleanup(mp, nxt_controller_cert_cleanup, task, ctrl_init.certs, rt); #endif +#if (NXT_HAVE_NJS) + ctrl_init.scripts = nxt_script_store_load(task, mp); + + nxt_mp_cleanup(mp, nxt_controller_script_cleanup, task, ctrl_init.scripts, + rt); +#endif + process->data.controller = ctrl_init; return NXT_OK; @@ -321,6 +338,13 @@ nxt_controller_start(nxt_task_t *task, nxt_process_data_t *data) } #endif +#if (NXT_HAVE_NJS) + if (init->scripts != NULL) { + nxt_script_info_init(task, init->scripts); + nxt_script_store_release(init->scripts); + } +#endif + json = &init->conf; if (json->start == NULL) { @@ -1047,9 +1071,19 @@ nxt_controller_process_request(nxt_task_t *task, nxt_controller_request_t *req) nxt_controller_response_t resp; #if (NXT_TLS) nxt_conf_value_t *certs; +#endif +#if (NXT_HAVE_NJS) + nxt_conf_value_t *scripts; +#endif +#if (NXT_TLS) static nxt_str_t certificates = nxt_string("certificates"); #endif + +#if (NXT_HAVE_NJS) + static nxt_str_t scripts_str = nxt_string("js_modules"); +#endif + static nxt_str_t config = nxt_string("config"); static nxt_str_t status = nxt_string("status"); @@ -1118,6 +1152,25 @@ nxt_controller_process_request(nxt_task_t *task, nxt_controller_request_t *req) return; } +#endif + +#if (NXT_HAVE_NJS) + + if (nxt_str_start(&path, "/js_modules", 11) + && (path.length == 11 || path.start[11] == '/')) + { + if (path.length == 11) { + path.length = 1; + + } else { + path.length -= 11; + path.start += 11; + } + + nxt_controller_process_script(task, req, &path); + return; + } + #endif if (nxt_str_start(&path, "/control/", 9)) { @@ -1143,6 +1196,9 @@ nxt_controller_process_request(nxt_task_t *task, nxt_controller_request_t *req) #if (NXT_TLS) count++; #endif +#if (NXT_HAVE_NJS) + count++; +#endif value = nxt_conf_create_object(c->mem_pool, count); if (nxt_slow_path(value == NULL)) { @@ -1160,6 +1216,15 @@ nxt_controller_process_request(nxt_task_t *task, nxt_controller_request_t *req) nxt_conf_set_member(value, &certificates, certs, i++); #endif +#if (NXT_HAVE_NJS) + scripts = nxt_script_info_get_all(c->mem_pool); + if (nxt_slow_path(scripts == NULL)) { + goto alloc_fail; + } + + nxt_conf_set_member(value, &scripts_str, scripts, i++); +#endif + nxt_conf_set_member(value, &config, nxt_controller_conf.root, i++); nxt_conf_set_member(value, &status, nxt_controller_status, i); @@ -1879,6 +1944,294 @@ nxt_controller_cert_in_use(nxt_str_t *name) #endif +#if (NXT_HAVE_NJS) + +static void +nxt_controller_process_script(nxt_task_t *task, + nxt_controller_request_t *req, nxt_str_t *path) +{ + u_char *p; + nxt_int_t ret; + nxt_str_t name; + nxt_conn_t *c; + nxt_script_t *script; + nxt_buf_mem_t *bm; + nxt_conf_value_t *value; + nxt_controller_response_t resp; + u_char error[NXT_MAX_ERROR_STR]; + + name.length = path->length - 1; + name.start = path->start + 1; + + p = memchr(name.start, '/', name.length); + + if (p != NULL) { + name.length = p - name.start; + + path->length -= p - path->start; + path->start = p; + + } else { + path = NULL; + } + + nxt_memzero(&resp, sizeof(nxt_controller_response_t)); + + c = req->conn; + + if (nxt_str_eq(&req->parser.method, "GET", 3)) { + + if (name.length != 0) { + value = nxt_script_info_get(&name); + if (value == NULL) { + goto script_not_found; + } + + if (path != NULL) { + value = nxt_conf_get_path(value, path); + if (value == NULL) { + goto not_found; + } + } + + } else { + value = nxt_script_info_get_all(c->mem_pool); + if (value == NULL) { + goto alloc_fail; + } + } + + resp.status = 200; + resp.conf = value; + + nxt_controller_response(task, req, &resp); + return; + } + + if (name.length == 0 || path != NULL) { + goto invalid_name; + } + + if (nxt_str_eq(&req->parser.method, "PUT", 3)) { + value = nxt_script_info_get(&name); + if (value != NULL) { + goto exists_script; + } + + bm = &c->read->mem; + + script = nxt_script_new(task, &name, bm->pos, + nxt_buf_mem_used_size(bm), error); + if (script == NULL) { + goto invalid_script; + } + + ret = nxt_script_info_save(&name, script); + + nxt_script_destroy(script); + + if (nxt_slow_path(ret != NXT_OK)) { + goto alloc_fail; + } + + nxt_script_store_get(task, &name, c->mem_pool, + nxt_controller_process_script_save, req); + return; + } + + if (nxt_str_eq(&req->parser.method, "DELETE", 6)) { + + if (nxt_controller_script_in_use(&name)) { + goto script_in_use; + } + + if (nxt_script_info_delete(&name) != NXT_OK) { + goto script_not_found; + } + + nxt_script_store_delete(task, &name, c->mem_pool); + + resp.status = 200; + resp.title = (u_char *) "JS module deleted."; + + nxt_controller_response(task, req, &resp); + return; + } + + resp.status = 405; + resp.title = (u_char *) "Invalid method."; + resp.offset = -1; + + nxt_controller_response(task, req, &resp); + return; + +invalid_name: + + resp.status = 400; + resp.title = (u_char *) "Invalid JS module name."; + resp.offset = -1; + + nxt_controller_response(task, req, &resp); + return; + +invalid_script: + + resp.status = 400; + resp.title = (u_char *) "Invalid JS module."; + resp.offset = -1; + + resp.detail.start = error; + resp.detail.length = nxt_strlen(error); + + nxt_controller_response(task, req, &resp); + return; + +exists_script: + + resp.status = 400; + resp.title = (u_char *) "JS module already exists."; + resp.offset = -1; + + nxt_controller_response(task, req, &resp); + return; + +script_in_use: + + resp.status = 400; + resp.title = (u_char *) "JS module is used in the configuration."; + resp.offset = -1; + + nxt_controller_response(task, req, &resp); + return; + +script_not_found: + + resp.status = 404; + resp.title = (u_char *) "JS module doesn't exist."; + resp.offset = -1; + + nxt_controller_response(task, req, &resp); + return; + +not_found: + + resp.status = 404; + resp.title = (u_char *) "Invalid path."; + resp.offset = -1; + + nxt_controller_response(task, req, &resp); + return; + +alloc_fail: + + resp.status = 500; + resp.title = (u_char *) "Memory allocation failed."; + resp.offset = -1; + + nxt_controller_response(task, req, &resp); +} + + +static void +nxt_controller_process_script_save(nxt_task_t *task, nxt_port_recv_msg_t *msg, + void *data) +{ + nxt_conn_t *c; + nxt_buf_mem_t *mbuf; + nxt_controller_request_t *req; + nxt_controller_response_t resp; + + req = data; + + nxt_memzero(&resp, sizeof(nxt_controller_response_t)); + + if (msg == NULL || msg->port_msg.type == _NXT_PORT_MSG_RPC_ERROR) { + resp.status = 500; + resp.title = (u_char *) "Failed to store script."; + + nxt_controller_response(task, req, &resp); + return; + } + + c = req->conn; + + mbuf = &c->read->mem; + + nxt_fd_write(msg->fd[0], mbuf->pos, nxt_buf_mem_used_size(mbuf)); + + nxt_fd_close(msg->fd[0]); + + nxt_memzero(&resp, sizeof(nxt_controller_response_t)); + + resp.status = 200; + resp.title = (u_char *) "JS module uploaded."; + + nxt_controller_response(task, req, &resp); +} + + +static nxt_bool_t +nxt_controller_script_in_use(nxt_str_t *name) +{ + uint32_t i, n; + nxt_str_t str; + nxt_conf_value_t *js_module, *element; + + static nxt_str_t js_module_path = nxt_string("/settings/js_module"); + + js_module = nxt_conf_get_path(nxt_controller_conf.root, + &js_module_path); + + if (js_module != NULL) { + + if (nxt_conf_type(js_module) == NXT_CONF_ARRAY) { + n = nxt_conf_array_elements_count(js_module); + + for (i = 0; i < n; i++) { + element = nxt_conf_get_array_element(js_module, i); + + nxt_conf_get_string(element, &str); + + if (nxt_strstr_eq(&str, name)) { + return 1; + } + } + + } else { + /* NXT_CONF_STRING */ + + nxt_conf_get_string(js_module, &str); + + if (nxt_strstr_eq(&str, name)) { + return 1; + } + } + } + + return 0; +} + + +static void +nxt_controller_script_cleanup(nxt_task_t *task, void *obj, void *data) +{ + pid_t main_pid; + nxt_array_t *scripts; + nxt_runtime_t *rt; + + scripts = obj; + rt = data; + + main_pid = rt->port_by_type[NXT_PROCESS_MAIN]->pid; + + if (nxt_pid == main_pid && scripts != NULL) { + nxt_script_store_release(scripts); + } +} + +#endif + + static void nxt_controller_conf_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) diff --git a/src/nxt_js.c b/src/nxt_js.c index 4327e848..df945db6 100644 --- a/src/nxt_js.c +++ b/src/nxt_js.c @@ -8,16 +8,72 @@ struct nxt_js_s { uint32_t index; - njs_vm_t *vm; }; +typedef struct { + nxt_str_t name; + nxt_str_t text; +} nxt_js_module_t; + + struct nxt_js_conf_s { nxt_mp_t *pool; njs_vm_t *vm; njs_uint_t protos; njs_external_t *proto; + nxt_str_t init; + nxt_array_t *modules; /* of nxt_js_module_t */ nxt_array_t *funcs; + uint8_t test; /* 1 bit */ +}; + + +njs_mod_t * +nxt_js_module_loader(njs_vm_t *vm, njs_external_ptr_t external, njs_str_t *name) +{ + nxt_str_t text; + nxt_uint_t i, n; + nxt_js_conf_t *jcf; + nxt_js_module_t *modules, *module; + + jcf = external; + + module = NULL; + + n = jcf->modules->nelts; + modules = jcf->modules->elts; + + for (i = 0; i < n; i++) { + if (nxt_strstr_eq(name, &modules[i].name)) { + module = &modules[i]; + break; + } + } + + if (module == NULL) { + return NULL; + } + + text.length = module->text.length; + + text.start = njs_mp_alloc(vm->mem_pool, text.length); + if (nxt_slow_path(text.start == NULL)) { + return NULL; + } + + nxt_memcpy(text.start, module->text.start, text.length); + + return njs_vm_compile_module(vm, name, &text.start, + &text.start[text.length]); +} + + +static njs_vm_ops_t nxt_js_ops = { + NULL, + NULL, + nxt_js_module_loader, + NULL, }; @@ -25,9 +81,8 @@ njs_int_t nxt_js_proto_id; nxt_js_conf_t * -nxt_js_conf_new(nxt_mp_t *mp) +nxt_js_conf_new(nxt_mp_t *mp, nxt_bool_t test) { - njs_vm_opt_t opts; nxt_js_conf_t *jcf; jcf = nxt_mp_zget(mp, sizeof(nxt_js_conf_t)); @@ -36,17 +91,15 @@ nxt_js_conf_new(nxt_mp_t *mp) } jcf->pool = mp; + jcf->test = test; - njs_vm_opt_init(&opts); - - jcf->vm = njs_vm_create(&opts); - if (nxt_slow_path(jcf->vm == NULL)) { + jcf->modules = nxt_array_create(mp, 4, sizeof(nxt_js_module_t)); + if (nxt_slow_path(jcf->modules == NULL)) { return NULL; } jcf->funcs = nxt_array_create(mp, 4, sizeof(nxt_str_t)); if (nxt_slow_path(jcf->funcs == NULL)) { - njs_vm_destroy(jcf->vm); return NULL; } @@ -69,6 +122,115 @@ nxt_js_set_proto(nxt_js_conf_t *jcf, njs_external_t *proto, njs_uint_t n) } +static njs_vm_t * +nxt_js_vm_create(nxt_js_conf_t *jcf) +{ + u_char *p; + size_t size; + nxt_uint_t i; + njs_vm_opt_t opts; + nxt_js_module_t *module, *mod; + + static nxt_str_t import_str = nxt_string("import"); + static nxt_str_t from_str = nxt_string("from"); + static nxt_str_t global_str = nxt_string("globalThis"); + + njs_vm_opt_init(&opts); + + opts.backtrace = 1; + + opts.file.start = (u_char *) "default"; + opts.file.length = 7; + + if (jcf->test || jcf->modules->nelts == 0) { + goto done; + } + + opts.ops = &nxt_js_ops; + opts.external = jcf; + + size = 0; + module = jcf->modules->elts; + + for (i = 0; i < jcf->modules->nelts; i++) { + mod = &module[i]; + + size += import_str.length + 1 + mod->name.length + 1 + + from_str.length + 2 + mod->name.length + 3; + + size += global_str.length + 1 + mod->name.length + 3 + + mod->name.length + 2; + } + + p = nxt_mp_nget(jcf->pool, size); + if (nxt_slow_path(p == NULL)) { + return NULL; + } + + jcf->init.length = size; + jcf->init.start = p; + + for (i = 0; i < jcf->modules->nelts; i++) { + mod = &module[i]; + + p = nxt_cpymem(p, import_str.start, import_str.length); + *p++ = ' '; + + p = nxt_cpymem(p, mod->name.start, mod->name.length); + *p++ = ' '; + + p = nxt_cpymem(p, from_str.start, from_str.length); + *p++ = ' '; + + *p++ = '\"'; + p = nxt_cpymem(p, mod->name.start, mod->name.length); + *p++ = '\"'; + *p++ = ';'; + *p++ = '\n'; + + p = nxt_cpymem(p, global_str.start, global_str.length); + *p++ = '.'; + + p = nxt_cpymem(p, mod->name.start, mod->name.length); + *p++ = ' '; + *p++ = '='; + *p++ = ' '; + + p = nxt_cpymem(p, mod->name.start, mod->name.length); + *p++ = ';'; + *p++ = '\n'; + } + +done: + + return njs_vm_create(&opts); +} + + +nxt_int_t +nxt_js_add_module(nxt_js_conf_t *jcf, nxt_str_t *name, nxt_str_t *text) +{ + nxt_js_module_t *module; + + module = nxt_array_add(jcf->modules); + if (nxt_slow_path(module == NULL)) { + return NXT_ERROR; + } + + module->name = *name; + + module->text.length = text->length; + module->text.start = nxt_mp_nget(jcf->pool, text->length); + if (nxt_slow_path(module->text.start == NULL)) { + return NXT_ERROR; + } + + nxt_memcpy(module->text.start, text->start, text->length); + + return NXT_OK; +} + + nxt_js_t * nxt_js_add_tpl(nxt_js_conf_t *jcf, nxt_str_t *str, nxt_bool_t strz) { @@ -113,8 +275,6 @@ nxt_js_add_tpl(nxt_js_conf_t *jcf, nxt_str_t *str, nxt_bool_t strz) return NULL; } - js->vm = jcf->vm; - func = nxt_array_add(jcf->funcs); if (nxt_slow_path(func == NULL)) { return NULL; @@ -138,7 +298,16 @@ nxt_js_compile(nxt_js_conf_t *jcf) nxt_str_t *func; nxt_uint_t i; - size = 2; + if (jcf->test) { + return NXT_OK; + } + + jcf->vm = nxt_js_vm_create(jcf); + if (nxt_slow_path(jcf->vm == NULL)) { + return NXT_ERROR; + } + + size = jcf->init.length + 2; func = jcf->funcs->elts; for (i = 0; i < jcf->funcs->nelts; i++) { @@ -150,7 +319,7 @@ nxt_js_compile(nxt_js_conf_t *jcf) return NXT_ERROR; } - p = start; + p = nxt_cpymem(start, jcf->init.start, jcf->init.length); *p++ = '['; func = jcf->funcs->elts; @@ -178,37 +347,43 @@ nxt_int_t nxt_js_test(nxt_js_conf_t *jcf, nxt_str_t *str, u_char *error) { u_char *start; - nxt_str_t err; + njs_vm_t *vm; njs_int_t ret; - njs_str_t res; + + vm = nxt_js_vm_create(jcf); + if (nxt_slow_path(vm == NULL)) { + return NXT_ERROR; + } start = nxt_mp_nget(jcf->pool, str->length); if (nxt_slow_path(start == NULL)) { - return NXT_ERROR; + goto fail; } nxt_memcpy(start, str->start, str->length); - ret = njs_vm_compile(jcf->vm, &start, start + str->length); + ret = njs_vm_compile(vm, &start, start + str->length); if (nxt_slow_path(ret != NJS_OK)) { - (void) njs_vm_retval_string(jcf->vm, &res); + (void) nxt_js_error(vm, error); + goto fail; + } + + njs_vm_destroy(vm); - err.start = res.start; - err.length = res.length; + return NXT_OK; - nxt_sprintf(error, error + NXT_MAX_ERROR_STR, "\"%V\"%Z", &err); +fail: - return NXT_ERROR; - } + njs_vm_destroy(vm); - return NXT_OK; + return NXT_ERROR; } nxt_int_t -nxt_js_call(nxt_task_t *task, nxt_js_cache_t *cache, nxt_js_t *js, - nxt_str_t *str, void *ctx) +nxt_js_call(nxt_task_t *task, nxt_js_conf_t *jcf, nxt_js_cache_t *cache, + nxt_js_t *js, nxt_str_t *str, void *ctx) { njs_vm_t *vm; njs_int_t rc, ret; @@ -227,7 +402,7 @@ nxt_js_call(nxt_task_t *task, nxt_js_cache_t *cache, nxt_js_t *js, vm = cache->vm; if (vm == NULL) { - vm = njs_vm_clone(js->vm, ctx); + vm = njs_vm_clone(jcf->vm, ctx); if (nxt_slow_path(vm == NULL)) { return NXT_ERROR; } @@ -314,3 +489,24 @@ nxt_js_release(nxt_js_cache_t *cache) njs_vm_destroy(cache->vm); } } + + +nxt_int_t +nxt_js_error(njs_vm_t *vm, u_char *error) +{ + njs_int_t ret; + njs_str_t res; + nxt_str_t err; + + ret = njs_vm_retval_string(vm, &res); + if (nxt_slow_path(ret != NJS_OK)) { + return NXT_ERROR; + } + + err.start = res.start; + err.length = res.length; + + nxt_sprintf(error, error + NXT_MAX_ERROR_STR, "\"%V\"%Z", &err); + + return NXT_OK; +} diff --git a/src/nxt_js.h b/src/nxt_js.h index 74d041ca..48f036b8 100644 --- a/src/nxt_js.h +++ b/src/nxt_js.h @@ -21,15 +21,20 @@ typedef struct { } nxt_js_cache_t; -nxt_js_conf_t *nxt_js_conf_new(nxt_mp_t *mp); +njs_mod_t *nxt_js_module_loader(njs_vm_t *vm, njs_external_ptr_t external, + njs_str_t *name); +nxt_js_conf_t *nxt_js_conf_new(nxt_mp_t *mp, nxt_bool_t test); void nxt_js_conf_release(nxt_js_conf_t *jcf); void nxt_js_set_proto(nxt_js_conf_t *jcf, njs_external_t *proto, nxt_uint_t n); +nxt_int_t nxt_js_add_module(nxt_js_conf_t *jcf, nxt_str_t *name, + nxt_str_t *text); nxt_js_t *nxt_js_add_tpl(nxt_js_conf_t *jcf, nxt_str_t *str, nxt_bool_t strz); nxt_int_t nxt_js_compile(nxt_js_conf_t *jcf); nxt_int_t nxt_js_test(nxt_js_conf_t *jcf, nxt_str_t *str, u_char *error); -nxt_int_t nxt_js_call(nxt_task_t *task, nxt_js_cache_t *cache, nxt_js_t *js, - nxt_str_t *str, void *ctx); +nxt_int_t nxt_js_call(nxt_task_t *task, nxt_js_conf_t *jcf, + nxt_js_cache_t *cache, nxt_js_t *js, nxt_str_t *str, void *ctx); void nxt_js_release(nxt_js_cache_t *cache); +nxt_int_t nxt_js_error(njs_vm_t *vm, u_char *error); extern njs_int_t nxt_js_proto_id; diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index ae62aff4..7cba08d4 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -14,6 +14,9 @@ #if (NXT_TLS) #include #endif +#if (NXT_HAVE_NJS) +#include +#endif #include @@ -607,6 +610,10 @@ static nxt_port_handlers_t nxt_main_process_port_handlers = { #if (NXT_TLS) .cert_get = nxt_cert_store_get_handler, .cert_delete = nxt_cert_store_delete_handler, +#endif +#if (NXT_HAVE_NJS) + .script_get = nxt_script_store_get_handler, + .script_delete = nxt_script_store_delete_handler, #endif .access_log = nxt_main_port_access_log_handler, .rpc_ready = nxt_port_rpc_handler, diff --git a/src/nxt_port.h b/src/nxt_port.h index eba8d06f..772fb41a 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -21,6 +21,8 @@ struct nxt_port_handlers_s { nxt_port_handler_t conf_store; nxt_port_handler_t cert_get; nxt_port_handler_t cert_delete; + nxt_port_handler_t script_get; + nxt_port_handler_t script_delete; nxt_port_handler_t access_log; /* File descriptor exchange. */ @@ -87,6 +89,8 @@ typedef enum { _NXT_PORT_MSG_CONF_STORE = nxt_port_handler_idx(conf_store), _NXT_PORT_MSG_CERT_GET = nxt_port_handler_idx(cert_get), _NXT_PORT_MSG_CERT_DELETE = nxt_port_handler_idx(cert_delete), + _NXT_PORT_MSG_SCRIPT_GET = nxt_port_handler_idx(script_get), + _NXT_PORT_MSG_SCRIPT_DELETE = nxt_port_handler_idx(script_delete), _NXT_PORT_MSG_ACCESS_LOG = nxt_port_handler_idx(access_log), _NXT_PORT_MSG_CHANGE_FILE = nxt_port_handler_idx(change_file), @@ -129,6 +133,8 @@ typedef enum { NXT_PORT_MSG_CONF_STORE = nxt_msg_last(_NXT_PORT_MSG_CONF_STORE), NXT_PORT_MSG_CERT_GET = nxt_msg_last(_NXT_PORT_MSG_CERT_GET), NXT_PORT_MSG_CERT_DELETE = nxt_msg_last(_NXT_PORT_MSG_CERT_DELETE), + NXT_PORT_MSG_SCRIPT_GET = nxt_msg_last(_NXT_PORT_MSG_SCRIPT_GET), + NXT_PORT_MSG_SCRIPT_DELETE = nxt_msg_last(_NXT_PORT_MSG_SCRIPT_DELETE), NXT_PORT_MSG_ACCESS_LOG = nxt_msg_last(_NXT_PORT_MSG_ACCESS_LOG), NXT_PORT_MSG_CHANGE_FILE = nxt_msg_last(_NXT_PORT_MSG_CHANGE_FILE), NXT_PORT_MSG_NEW_PORT = nxt_msg_last(_NXT_PORT_MSG_NEW_PORT), diff --git a/src/nxt_process.h b/src/nxt_process.h index 16d6110c..42fd1bed 100644 --- a/src/nxt_process.h +++ b/src/nxt_process.h @@ -29,6 +29,9 @@ typedef struct { #if (NXT_TLS) nxt_array_t *certs; #endif +#if (NXT_HAVE_NJS) + nxt_array_t *scripts; +#endif } nxt_controller_init_t; diff --git a/src/nxt_router.c b/src/nxt_router.c index c4e29e3a..d089cfb8 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -11,6 +11,9 @@ #if (NXT_TLS) #include #endif +#if (NXT_HAVE_NJS) +#include +#endif #include #include #include @@ -55,6 +58,17 @@ typedef struct { #endif +#if (NXT_HAVE_NJS) + +typedef struct { + nxt_str_t name; + nxt_router_temp_conf_t *temp_conf; + nxt_queue_link_t link; +} nxt_router_js_module_t; + +#endif + + typedef struct { nxt_str_t *name; nxt_socket_conf_t *socket_conf; @@ -139,6 +153,12 @@ static nxt_int_t nxt_router_conf_tls_insert(nxt_router_temp_conf_t *tmcf, nxt_conf_value_t *value, nxt_socket_conf_t *skcf, nxt_tls_init_t *tls_init, nxt_bool_t last); #endif +#if (NXT_HAVE_NJS) +static void nxt_router_js_module_rpc_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg, void *data); +static nxt_int_t nxt_router_js_module_insert(nxt_router_temp_conf_t *tmcf, + nxt_conf_value_t *value); +#endif static void nxt_router_app_rpc_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_app_t *app); static void nxt_router_app_prefork_ready(nxt_task_t *task, @@ -1100,6 +1120,10 @@ nxt_router_temp_conf(nxt_task_t *task) nxt_queue_init(&tmcf->tls); #endif +#if (NXT_HAVE_NJS) + nxt_queue_init(&tmcf->js_modules); +#endif + nxt_queue_init(&tmcf->apps); nxt_queue_init(&tmcf->previous); @@ -1154,6 +1178,9 @@ nxt_router_conf_apply(nxt_task_t *task, void *obj, void *data) #if (NXT_TLS) nxt_router_tlssock_t *tls; #endif +#if (NXT_HAVE_NJS) + nxt_router_js_module_t *js_module; +#endif tmcf = obj; @@ -1184,6 +1211,27 @@ nxt_router_conf_apply(nxt_task_t *task, void *obj, void *data) } #endif +#if (NXT_HAVE_NJS) + qlk = nxt_queue_last(&tmcf->js_modules); + + if (qlk != nxt_queue_head(&tmcf->js_modules)) { + nxt_queue_remove(qlk); + + js_module = nxt_queue_link_data(qlk, nxt_router_js_module_t, link); + + nxt_script_store_get(task, &js_module->name, tmcf->mem_pool, + nxt_router_js_module_rpc_handler, js_module); + return; + } +#endif + + rtcf = tmcf->router_conf; + + ret = nxt_tstr_state_done(rtcf->tstr_state, NULL); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + nxt_queue_each(app, &tmcf->apps, nxt_app_t, link) { if (nxt_router_app_need_start(app)) { @@ -1193,8 +1241,6 @@ nxt_router_conf_apply(nxt_task_t *task, void *obj, void *data) } nxt_queue_loop; - rtcf = tmcf->router_conf; - if (rtcf->access_log != NULL && rtcf->access_log->fd == -1) { nxt_router_access_log_open(task, tmcf); return; @@ -1569,6 +1615,9 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, #if (NXT_TLS) nxt_tls_init_t *tls_init; nxt_conf_value_t *certificate; +#endif +#if (NXT_HAVE_NJS) + nxt_conf_value_t *js_module; #endif nxt_conf_value_t *root, *conf, *http, *value, *websocket; nxt_conf_value_t *applications, *application; @@ -1592,6 +1641,9 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, static nxt_str_t conf_cache_path = nxt_string("/tls/session/cache_size"); static nxt_str_t conf_timeout_path = nxt_string("/tls/session/timeout"); static nxt_str_t conf_tickets = nxt_string("/tls/session/tickets"); +#endif +#if (NXT_HAVE_NJS) + static nxt_str_t js_module_path = nxt_string("/settings/js_module"); #endif static nxt_str_t static_path = nxt_string("/settings/http/static"); static nxt_str_t websocket_path = nxt_string("/settings/http/websocket"); @@ -2064,11 +2116,34 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, } } - ret = nxt_tstr_state_done(rtcf->tstr_state, NULL); - if (nxt_slow_path(ret != NXT_OK)) { - goto fail; +#if (NXT_HAVE_NJS) + js_module = nxt_conf_get_path(root, &js_module_path); + + if (js_module != NULL) { + if (nxt_conf_type(js_module) == NXT_CONF_ARRAY) { + n = nxt_conf_array_elements_count(js_module); + + for (i = 0; i < n; i++) { + value = nxt_conf_get_array_element(js_module, i); + + ret = nxt_router_js_module_insert(tmcf, value); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + } + + } else { + /* NXT_CONF_STRING */ + + ret = nxt_router_js_module_insert(tmcf, js_module); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + } } +#endif + nxt_queue_add(&deleting_sockets, &router->sockets); nxt_queue_init(&router->sockets); @@ -2120,6 +2195,79 @@ nxt_router_conf_tls_insert(nxt_router_temp_conf_t *tmcf, #endif +#if (NXT_HAVE_NJS) + +static void +nxt_router_js_module_rpc_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, + void *data) +{ + nxt_int_t ret; + nxt_str_t text; + nxt_router_conf_t *rtcf; + nxt_router_temp_conf_t *tmcf; + nxt_router_js_module_t *js_module; + + nxt_debug(task, "auto module rpc handler"); + + js_module = data; + tmcf = js_module->temp_conf; + + if (msg == NULL || msg->port_msg.type == _NXT_PORT_MSG_RPC_ERROR) { + goto fail; + } + + rtcf = tmcf->router_conf; + + ret = nxt_script_file_read(msg->fd[0], &text); + + nxt_fd_close(msg->fd[0]); + + if (nxt_slow_path(ret == NXT_ERROR)) { + goto fail; + } + + if (text.length > 0) { + ret = nxt_js_add_module(rtcf->tstr_state->jcf, &js_module->name, &text); + + nxt_free(text.start); + + if (nxt_slow_path(ret == NXT_ERROR)) { + goto fail; + } + } + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + nxt_router_conf_apply, task, tmcf, NULL); + return; + +fail: + + nxt_router_conf_error(task, tmcf); +} + + +static nxt_int_t +nxt_router_js_module_insert(nxt_router_temp_conf_t *tmcf, + nxt_conf_value_t *value) +{ + nxt_router_js_module_t *js_module; + + js_module = nxt_mp_get(tmcf->mem_pool, sizeof(nxt_router_js_module_t)); + if (nxt_slow_path(js_module == NULL)) { + return NXT_ERROR; + } + + js_module->temp_conf = tmcf; + nxt_conf_get_string(value, &js_module->name); + + nxt_queue_insert_tail(&tmcf->js_modules, &js_module->link); + + return NXT_OK; +} + +#endif + + static nxt_int_t nxt_router_conf_process_static(nxt_task_t *task, nxt_router_conf_t *rtcf, nxt_conf_value_t *conf) diff --git a/src/nxt_router.h b/src/nxt_router.h index 6f0ba5ad..b14f8410 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -74,6 +74,10 @@ typedef struct { nxt_queue_t tls; /* of nxt_router_tlssock_t */ #endif +#if (NXT_HAVE_NJS) + nxt_queue_t js_modules; +#endif + nxt_queue_t apps; /* of nxt_app_t */ nxt_queue_t previous; /* of nxt_app_t */ diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index 1da52da0..96f801fb 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -906,6 +906,23 @@ nxt_runtime_conf_init(nxt_task_t *task, nxt_runtime_t *rt) "mkdir(%s) failed %E", file_name.start, nxt_errno); } + ret = nxt_file_name_create(rt->mem_pool, &file_name, "%s%sscripts/%Z", + rt->state, slash); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + ret = mkdir((char *) file_name.start, S_IRWXU); + + if (nxt_fast_path(ret == 0 || nxt_errno == EEXIST)) { + rt->scripts.length = file_name.len; + rt->scripts.start = file_name.start; + + } else { + nxt_alert(task, "Unable to create scripts storage directory: " + "mkdir(%s) failed %E", file_name.start, nxt_errno); + } + control.length = nxt_strlen(rt->control); control.start = (u_char *) rt->control; diff --git a/src/nxt_runtime.h b/src/nxt_runtime.h index 687914f0..66ec0106 100644 --- a/src/nxt_runtime.h +++ b/src/nxt_runtime.h @@ -74,6 +74,7 @@ struct nxt_runtime_s { const char *tmp; nxt_str_t certs; + nxt_str_t scripts; nxt_queue_t engines; /* of nxt_event_engine_t */ diff --git a/src/nxt_script.c b/src/nxt_script.c new file mode 100644 index 00000000..70045a22 --- /dev/null +++ b/src/nxt_script.c @@ -0,0 +1,709 @@ + +/* + * Copyright (C) NGINX, Inc. + * Copyright (C) Zhidao HONG + */ + +#include +#include +#include +#include + + +struct nxt_script_s { + nxt_str_t text; +}; + + +typedef struct { + nxt_str_t name; + nxt_conf_value_t *value; + nxt_mp_t *mp; +} nxt_script_info_t; + + +typedef struct { + nxt_str_t name; + nxt_fd_t fd; +} nxt_script_item_t; + + +static nxt_script_t *nxt_script_get(nxt_task_t *task, nxt_str_t *name, + nxt_fd_t fd); +static nxt_conf_value_t *nxt_script_details(nxt_mp_t *mp, nxt_script_t *cert); +static void nxt_script_buf_completion(nxt_task_t *task, void *obj, void *data); + + +static nxt_lvlhsh_t nxt_script_info; + + +static njs_vm_ops_t nxt_js_ops = { + NULL, + NULL, + nxt_js_module_loader, + NULL, +}; + + +nxt_script_t * +nxt_script_new(nxt_task_t *task, nxt_str_t *name, u_char *data, size_t size, + u_char *error) +{ + u_char *start; + njs_vm_t *vm; + njs_str_t mod_name; + njs_mod_t *mod; + njs_vm_opt_t opts; + nxt_script_t *script; + + njs_vm_opt_init(&opts); + + opts.backtrace = 1; + + opts.file.start = (u_char *) "default"; + opts.file.length = 7; + + opts.ops = &nxt_js_ops; + + vm = njs_vm_create(&opts); + if (nxt_slow_path(vm == NULL)) { + return NULL; + } + + mod_name.length = name->length; + mod_name.start = name->start; + + start = data; + + mod = njs_vm_compile_module(vm, &mod_name, &start, start + size); + + if (nxt_slow_path(mod == NULL)) { + (void) nxt_js_error(vm, error); + nxt_alert(task, "JS compile module(%V) failed: %s", name, error); + + goto fail; + } + + script = nxt_zalloc(sizeof(nxt_script_t) + size); + if (nxt_slow_path(script == NULL)) { + goto fail; + } + + script->text.length = size; + script->text.start = (u_char *) script + sizeof(nxt_script_t); + + nxt_memcpy(script->text.start, data, size); + + njs_vm_destroy(vm); + + return script; + +fail: + + njs_vm_destroy(vm); + + return NULL; +} + + +static nxt_script_t * +nxt_script_get(nxt_task_t *task, nxt_str_t *name, nxt_fd_t fd) +{ + nxt_int_t ret; + nxt_str_t text; + nxt_script_t *script; + u_char error[NXT_MAX_ERROR_STR]; + + ret = nxt_script_file_read(fd, &text); + if (nxt_slow_path(ret != NXT_OK)) { + return NULL; + } + + script = nxt_script_new(task, name, text.start, text.length, error); + + nxt_free(text.start); + + return script; +} + + +void +nxt_script_destroy(nxt_script_t *script) +{ + nxt_free(script); +} + + +static nxt_int_t +nxt_script_info_hash_test(nxt_lvlhsh_query_t *lhq, void *data) +{ + nxt_script_info_t *info; + + info = data; + + if (nxt_strcasestr_eq(&lhq->key, &info->name)) { + return NXT_OK; + } + + return NXT_DECLINED; +} + + +static const nxt_lvlhsh_proto_t nxt_script_info_hash_proto + nxt_aligned(64) = +{ + NXT_LVLHSH_DEFAULT, + nxt_script_info_hash_test, + nxt_lvlhsh_alloc, + nxt_lvlhsh_free, +}; + + +void +nxt_script_info_init(nxt_task_t *task, nxt_array_t *scripts) +{ + uint32_t i; + nxt_script_t *script; + nxt_script_item_t *item; + + item = scripts->elts; + + for (i = 0; i < scripts->nelts; i++) { + script = nxt_script_get(task, &item->name, item->fd); + + if (nxt_slow_path(script == NULL)) { + continue; + } + + (void) nxt_script_info_save(&item->name, script); + + nxt_script_destroy(script); + + item++; + } +} + + +nxt_int_t +nxt_script_info_save(nxt_str_t *name, nxt_script_t *script) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_conf_value_t *value; + nxt_script_info_t *info; + nxt_lvlhsh_query_t lhq; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + info = nxt_mp_get(mp, sizeof(nxt_script_info_t)); + if (nxt_slow_path(info == NULL)) { + goto fail; + } + + name = nxt_str_dup(mp, &info->name, name); + if (nxt_slow_path(name == NULL)) { + goto fail; + } + + value = nxt_script_details(mp, script); + if (nxt_slow_path(value == NULL)) { + goto fail; + } + + info->mp = mp; + info->value = value; + + lhq.key_hash = nxt_djb_hash(name->start, name->length); + lhq.replace = 1; + lhq.key = *name; + lhq.value = info; + lhq.proto = &nxt_script_info_hash_proto; + + ret = nxt_lvlhsh_insert(&nxt_script_info, &lhq); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + + if (lhq.value != info) { + info = lhq.value; + nxt_mp_destroy(info->mp); + } + + return NXT_OK; + +fail: + + nxt_mp_destroy(mp); + return NXT_ERROR; +} + + +nxt_conf_value_t * +nxt_script_info_get(nxt_str_t *name) +{ + nxt_int_t ret; + nxt_script_info_t *info; + nxt_lvlhsh_query_t lhq; + + lhq.key_hash = nxt_djb_hash(name->start, name->length); + lhq.key = *name; + lhq.proto = &nxt_script_info_hash_proto; + + ret = nxt_lvlhsh_find(&nxt_script_info, &lhq); + if (ret != NXT_OK) { + return NULL; + } + + info = lhq.value; + + return info->value; +} + + +nxt_conf_value_t * +nxt_script_info_get_all(nxt_mp_t *mp) +{ + uint32_t i; + nxt_conf_value_t *all; + nxt_script_info_t *info; + nxt_lvlhsh_each_t lhe; + + nxt_lvlhsh_each_init(&lhe, &nxt_script_info_hash_proto); + + for (i = 0; /* void */; i++) { + info = nxt_lvlhsh_each(&nxt_script_info, &lhe); + + if (info == NULL) { + break; + } + } + + all = nxt_conf_create_object(mp, i); + if (nxt_slow_path(all == NULL)) { + return NULL; + } + + nxt_lvlhsh_each_init(&lhe, &nxt_script_info_hash_proto); + + for (i = 0; /* void */; i++) { + info = nxt_lvlhsh_each(&nxt_script_info, &lhe); + + if (info == NULL) { + break; + } + + nxt_conf_set_member(all, &info->name, info->value, i); + } + + return all; +} + + +static nxt_conf_value_t * +nxt_script_details(nxt_mp_t *mp, nxt_script_t *script) +{ + nxt_conf_value_t *value; + + value = nxt_conf_create_object(mp, 0); + if (nxt_slow_path(value == NULL)) { + return NULL; + } + + nxt_conf_set_string_dup(value, mp, &script->text); + + return value; +} + + +nxt_int_t +nxt_script_info_delete(nxt_str_t *name) +{ + nxt_int_t ret; + nxt_script_info_t *info; + nxt_lvlhsh_query_t lhq; + + lhq.key_hash = nxt_djb_hash(name->start, name->length); + lhq.key = *name; + lhq.proto = &nxt_script_info_hash_proto; + + ret = nxt_lvlhsh_delete(&nxt_script_info, &lhq); + + if (ret == NXT_OK) { + info = lhq.value; + nxt_mp_destroy(info->mp); + } + + return ret; +} + + +nxt_array_t * +nxt_script_store_load(nxt_task_t *task, nxt_mp_t *mp) +{ + DIR *dir; + size_t size, alloc; + u_char *buf, *p; + nxt_str_t name; + nxt_int_t ret; + nxt_file_t file; + nxt_array_t *scripts; + nxt_runtime_t *rt; + struct dirent *de; + nxt_script_item_t *item; + + rt = task->thread->runtime; + + if (nxt_slow_path(rt->scripts.start == NULL)) { + nxt_alert(task, "no scripts storage directory"); + return NULL; + } + + scripts = nxt_array_create(mp, 16, sizeof(nxt_script_item_t)); + if (nxt_slow_path(scripts == NULL)) { + return NULL; + } + + buf = NULL; + alloc = 0; + + dir = opendir((char *) rt->scripts.start); + if (nxt_slow_path(dir == NULL)) { + nxt_alert(task, "opendir(\"%s\") failed %E", + rt->scripts.start, nxt_errno); + goto fail; + } + + for ( ;; ) { + de = readdir(dir); + if (de == NULL) { + break; + } + + nxt_debug(task, "readdir(\"%s\"): \"%s\"", + rt->scripts.start, de->d_name); + + name.length = nxt_strlen(de->d_name); + name.start = (u_char *) de->d_name; + + if (nxt_str_eq(&name, ".", 1) || nxt_str_eq(&name, "..", 2)) { + continue; + } + + item = nxt_array_add(scripts); + if (nxt_slow_path(item == NULL)) { + goto fail; + } + + item->fd = -1; + + size = rt->scripts.length + name.length + 1; + + if (size > alloc) { + size += 32; + + p = nxt_realloc(buf, size); + if (p == NULL) { + goto fail; + } + + alloc = size; + buf = p; + } + + p = nxt_cpymem(buf, rt->scripts.start, rt->scripts.length); + p = nxt_cpymem(p, name.start, name.length + 1); + + nxt_memzero(&file, sizeof(nxt_file_t)); + + file.name = buf; + + ret = nxt_file_open(task, &file, NXT_FILE_RDONLY, NXT_FILE_OPEN, + NXT_FILE_OWNER_ACCESS); + + if (nxt_slow_path(ret != NXT_OK)) { + nxt_array_remove_last(scripts); + continue; + } + + item->fd = file.fd; + + if (nxt_slow_path(nxt_str_dup(mp, &item->name, &name) == NULL)) { + goto fail; + } + } + + if (buf != NULL) { + nxt_free(buf); + } + + (void) closedir(dir); + + return scripts; + +fail: + + if (buf != NULL) { + nxt_free(buf); + } + + if (dir != NULL) { + (void) closedir(dir); + } + + nxt_script_store_release(scripts); + + return NULL; +} + + +void +nxt_script_store_release(nxt_array_t *scripts) +{ + uint32_t i; + nxt_script_item_t *item; + + item = scripts->elts; + + for (i = 0; i < scripts->nelts; i++) { + nxt_fd_close(item[i].fd); + } + + nxt_array_destroy(scripts); +} + + +void +nxt_script_store_get(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp, + nxt_port_rpc_handler_t handler, void *ctx) +{ + uint32_t stream; + nxt_int_t ret; + nxt_buf_t *b; + nxt_port_t *main_port, *recv_port; + nxt_runtime_t *rt; + + b = nxt_buf_mem_alloc(mp, name->length + 1, 0); + if (nxt_slow_path(b == NULL)) { + goto fail; + } + + nxt_mp_retain(mp); + b->completion_handler = nxt_script_buf_completion; + + nxt_buf_cpystr(b, name); + *b->mem.free++ = '\0'; + + rt = task->thread->runtime; + main_port = rt->port_by_type[NXT_PROCESS_MAIN]; + recv_port = rt->port_by_type[rt->type]; + + stream = nxt_port_rpc_register_handler(task, recv_port, handler, handler, + -1, ctx); + if (nxt_slow_path(stream == 0)) { + goto fail; + } + + ret = nxt_port_socket_write(task, main_port, NXT_PORT_MSG_SCRIPT_GET, -1, + stream, recv_port->id, b); + + if (nxt_slow_path(ret != NXT_OK)) { + nxt_port_rpc_cancel(task, recv_port, stream); + goto fail; + } + + return; + +fail: + + handler(task, NULL, ctx); +} + + +static void +nxt_script_buf_completion(nxt_task_t *task, void *obj, void *data) +{ + nxt_mp_t *mp; + nxt_buf_t *b; + + b = obj; + mp = b->data; + nxt_assert(b->next == NULL); + + nxt_mp_free(mp, b); + nxt_mp_release(mp); +} + + +void +nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + u_char *p; + nxt_int_t ret; + nxt_str_t name; + nxt_file_t file; + nxt_port_t *port; + nxt_runtime_t *rt; + nxt_port_msg_type_t type; + + port = nxt_runtime_port_find(task->thread->runtime, msg->port_msg.pid, + msg->port_msg.reply_port); + + if (nxt_slow_path(port == NULL)) { + nxt_alert(task, "process port not found (pid %PI, reply_port %d)", + msg->port_msg.pid, msg->port_msg.reply_port); + return; + } + + if (nxt_slow_path(port->type != NXT_PROCESS_CONTROLLER + && port->type != NXT_PROCESS_ROUTER)) + { + nxt_alert(task, "process %PI cannot store scripts", + msg->port_msg.pid); + return; + } + + nxt_memzero(&file, sizeof(nxt_file_t)); + + file.fd = -1; + type = NXT_PORT_MSG_RPC_ERROR; + + rt = task->thread->runtime; + + if (nxt_slow_path(rt->certs.start == NULL)) { + nxt_alert(task, "no scripts storage directory"); + goto error; + } + + name.start = msg->buf->mem.pos; + name.length = nxt_strlen(name.start); + + file.name = nxt_malloc(rt->scripts.length + name.length + 1); + if (nxt_slow_path(file.name == NULL)) { + goto error; + } + + p = nxt_cpymem(file.name, rt->scripts.start, rt->scripts.length); + p = nxt_cpymem(p, name.start, name.length + 1); + + ret = nxt_file_open(task, &file, NXT_FILE_RDWR, NXT_FILE_CREATE_OR_OPEN, + NXT_FILE_OWNER_ACCESS); + + nxt_free(file.name); + + if (nxt_fast_path(ret == NXT_OK)) { + type = NXT_PORT_MSG_RPC_READY_LAST | NXT_PORT_MSG_CLOSE_FD; + } + +error: + + (void) nxt_port_socket_write(task, port, type, file.fd, + msg->port_msg.stream, 0, NULL); +} + + +void +nxt_script_store_delete(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp) +{ + nxt_buf_t *b; + nxt_port_t *main_port; + nxt_runtime_t *rt; + + b = nxt_buf_mem_alloc(mp, name->length + 1, 0); + + if (nxt_fast_path(b != NULL)) { + nxt_buf_cpystr(b, name); + *b->mem.free++ = '\0'; + + rt = task->thread->runtime; + main_port = rt->port_by_type[NXT_PROCESS_MAIN]; + + (void) nxt_port_socket_write(task, main_port, + NXT_PORT_MSG_SCRIPT_DELETE, -1, 0, 0, b); + } +} + + +void +nxt_script_store_delete_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + u_char *p; + nxt_str_t name; + nxt_port_t *ctl_port; + nxt_runtime_t *rt; + nxt_file_name_t *path; + + rt = task->thread->runtime; + ctl_port = rt->port_by_type[NXT_PROCESS_CONTROLLER]; + + if (nxt_slow_path(ctl_port == NULL)) { + nxt_alert(task, "controller port not found"); + return; + } + + if (nxt_slow_path(nxt_recv_msg_cmsg_pid(msg) != ctl_port->pid)) { + nxt_alert(task, "process %PI cannot delete scripts", + nxt_recv_msg_cmsg_pid(msg)); + return; + } + + if (nxt_slow_path(rt->scripts.start == NULL)) { + nxt_alert(task, "no scripts storage directory"); + return; + } + + name.start = msg->buf->mem.pos; + name.length = nxt_strlen(name.start); + + path = nxt_malloc(rt->scripts.length + name.length + 1); + + if (nxt_fast_path(path != NULL)) { + p = nxt_cpymem(path, rt->scripts.start, rt->scripts.length); + p = nxt_cpymem(p, name.start, name.length + 1); + + (void) nxt_file_delete(path); + + nxt_free(path); + } +} + + +nxt_int_t +nxt_script_file_read(nxt_fd_t fd, nxt_str_t *str) +{ + ssize_t n; + nxt_int_t ret; + nxt_file_t file; + nxt_file_info_t fi; + + nxt_memzero(&file, sizeof(nxt_file_t)); + + file.fd = fd; + + ret = nxt_file_info(&file, &fi); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + if (nxt_slow_path(!nxt_is_file(&fi))) { + nxt_str_null(str); + return NXT_DECLINED; + } + + str->length = nxt_file_size(&fi); + str->start = nxt_malloc(str->length); + if (nxt_slow_path(str->start == NULL)) { + return NXT_ERROR; + } + + n = nxt_file_read(&file, str->start, str->length, 0); + + if (nxt_slow_path(n != (ssize_t) str->length)) { + nxt_free(str->start); + return NXT_ERROR; + } + + return NXT_OK; +} diff --git a/src/nxt_script.h b/src/nxt_script.h new file mode 100644 index 00000000..ffefc108 --- /dev/null +++ b/src/nxt_script.h @@ -0,0 +1,37 @@ + +/* + * Copyright (C) NGINX, Inc. + * Copyright (C) Zhidao HONG + */ + +#ifndef _NXT_SCRIPT_INCLUDED_ +#define _NXT_SCRIPT_INCLUDED_ + + +typedef struct nxt_script_s nxt_script_t; + +nxt_script_t *nxt_script_new(nxt_task_t *task, nxt_str_t *name, u_char *data, + size_t size, u_char *error); +void nxt_script_destroy(nxt_script_t *script); + +void nxt_script_info_init(nxt_task_t *task, nxt_array_t *scripts); +nxt_int_t nxt_script_info_save(nxt_str_t *name, nxt_script_t *script); +nxt_conf_value_t *nxt_script_info_get(nxt_str_t *name); +nxt_conf_value_t *nxt_script_info_get_all(nxt_mp_t *mp); +nxt_int_t nxt_script_info_delete(nxt_str_t *name); + +nxt_array_t *nxt_script_store_load(nxt_task_t *task, nxt_mp_t *mem_pool); +void nxt_script_store_release(nxt_array_t *scripts); + +void nxt_script_store_get(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp, + nxt_port_rpc_handler_t handler, void *ctx); +void nxt_script_store_delete(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp); + +void nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +void nxt_script_store_delete_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); + +nxt_int_t nxt_script_file_read(nxt_fd_t fd, nxt_str_t *str); + + +#endif /* _NXT_SCRIPT_INCLUDED_ */ diff --git a/src/nxt_tstr.c b/src/nxt_tstr.c index c439696e..516415d9 100644 --- a/src/nxt_tstr.c +++ b/src/nxt_tstr.c @@ -70,7 +70,7 @@ nxt_tstr_state_new(nxt_mp_t *mp, nxt_bool_t test) } #if (NXT_HAVE_NJS) - state->jcf = nxt_js_conf_new(mp); + state->jcf = nxt_js_conf_new(mp, test); if (nxt_slow_path(state->jcf == NULL)) { return NULL; } @@ -273,7 +273,8 @@ nxt_tstr_query(nxt_task_t *task, nxt_tstr_query_t *query, nxt_tstr_t *tstr, } else { #if (NXT_HAVE_NJS) - ret = nxt_js_call(task, &query->cache->js, tstr->u.js, val, query->ctx); + ret = nxt_js_call(task, query->state->jcf, &query->cache->js, + tstr->u.js, val, query->ctx); if (nxt_slow_path(ret != NXT_OK)) { query->failed = 1; -- cgit