From c4f2a58aaf1096e2c25a99ff204d7be1beec8fb5 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Tue, 13 Oct 2020 01:37:39 +0300 Subject: Version bump. --- version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version b/version index a3b9c907..2ad4b832 100644 --- a/version +++ b/version @@ -1,5 +1,5 @@ # Copyright (C) NGINX, Inc. -NXT_VERSION=1.20.0 -NXT_VERNUM=12000 +NXT_VERSION=1.21.0 +NXT_VERNUM=12100 -- cgit From 091916614539c01e571486f2a55c68f87bd435bf Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Tue, 13 Oct 2020 01:37:39 +0300 Subject: Fixed building with Python 3.9. PyUnicode_GET_SIZE() in deprecated since 3.3 and will be removed in 3.12. In version 3.9 it was explicitly marked by deprecation warning causing compilation error with Unit. PyUnicode_GET_LENGTH() must be used instead. --- src/python/nxt_python.h | 1 + src/python/nxt_python_wsgi.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/python/nxt_python.h b/src/python/nxt_python.h index 3211026b..3dd04e31 100644 --- a/src/python/nxt_python.h +++ b/src/python/nxt_python.h @@ -28,6 +28,7 @@ #define PyBytes_AS_STRING PyString_AS_STRING #define PyUnicode_InternInPlace PyString_InternInPlace #define PyUnicode_AsUTF8 PyString_AS_STRING +#define PyUnicode_GET_LENGTH PyUnicode_GET_SIZE #endif #if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 5 diff --git a/src/python/nxt_python_wsgi.c b/src/python/nxt_python_wsgi.c index 97030cd3..8f692941 100644 --- a/src/python/nxt_python_wsgi.c +++ b/src/python/nxt_python_wsgi.c @@ -831,7 +831,7 @@ nxt_py_start_resp(PyObject *self, PyObject *args) fields_size += PyBytes_GET_SIZE(string); } else if (PyUnicode_Check(string)) { - fields_size += PyUnicode_GET_SIZE(string); + fields_size += PyUnicode_GET_LENGTH(string); } else { return PyErr_Format(PyExc_TypeError, @@ -843,7 +843,7 @@ nxt_py_start_resp(PyObject *self, PyObject *args) fields_size += PyBytes_GET_SIZE(string); } else if (PyUnicode_Check(string)) { - fields_size += PyUnicode_GET_SIZE(string); + fields_size += PyUnicode_GET_LENGTH(string); } else { return PyErr_Format(PyExc_TypeError, -- cgit From edafa954d42a83d2f6c81a4da45ddfc0678601b2 Mon Sep 17 00:00:00 2001 From: Igor Sysoev Date: Tue, 13 Oct 2020 12:56:56 +0300 Subject: Reordering declarations. --- src/nxt_conf_validation.c | 613 ++++++++++++++++++++++++---------------------- 1 file changed, 317 insertions(+), 296 deletions(-) diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 4364057b..c01cee62 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -170,21 +170,60 @@ static nxt_int_t nxt_conf_vldt_clone_gidmap(nxt_conf_validation_t *vldt, nxt_conf_value_t *value); #endif -static nxt_conf_vldt_object_t nxt_conf_vldt_websocket_members[] = { - { nxt_string("read_timeout"), - NXT_CONF_VLDT_INTEGER, + +static nxt_conf_vldt_object_t nxt_conf_vldt_setting_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_websocket_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_static_members[]; +#if (NXT_TLS) +static nxt_conf_vldt_object_t nxt_conf_vldt_tls_members[]; +#endif +static nxt_conf_vldt_object_t nxt_conf_vldt_match_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_php_common_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_php_options_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_common_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_app_limits_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_app_processes_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[]; +#if (NXT_HAVE_ISOLATION_ROOTFS) +static nxt_conf_vldt_object_t nxt_conf_vldt_app_automount_members[]; +#endif + + +static nxt_conf_vldt_object_t nxt_conf_vldt_root_members[] = { + { nxt_string("settings"), + NXT_CONF_VLDT_OBJECT, 0, - NULL, - NULL }, + &nxt_conf_vldt_object, + (void *) &nxt_conf_vldt_setting_members }, - { nxt_string("keepalive_interval"), - NXT_CONF_VLDT_INTEGER, + { nxt_string("listeners"), + NXT_CONF_VLDT_OBJECT, 0, - NULL, + &nxt_conf_vldt_object_iterator, + (void *) &nxt_conf_vldt_listener }, + + { nxt_string("routes"), + NXT_CONF_VLDT_ARRAY | NXT_CONF_VLDT_OBJECT, + 0, + &nxt_conf_vldt_routes, NULL }, - { nxt_string("max_frame_size"), - NXT_CONF_VLDT_INTEGER, + { nxt_string("applications"), + NXT_CONF_VLDT_OBJECT, + 0, + &nxt_conf_vldt_object_iterator, + (void *) &nxt_conf_vldt_app }, + + { nxt_string("upstreams"), + NXT_CONF_VLDT_OBJECT, + 0, + &nxt_conf_vldt_object_iterator, + (void *) &nxt_conf_vldt_upstream }, + + { nxt_string("access_log"), + NXT_CONF_VLDT_STRING, 0, NULL, NULL }, @@ -193,12 +232,12 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_websocket_members[] = { }; -static nxt_conf_vldt_object_t nxt_conf_vldt_static_members[] = { - { nxt_string("mime_types"), +static nxt_conf_vldt_object_t nxt_conf_vldt_setting_members[] = { + { nxt_string("http"), NXT_CONF_VLDT_OBJECT, 0, - &nxt_conf_vldt_mtypes, - NULL }, + &nxt_conf_vldt_object, + (void *) &nxt_conf_vldt_http_members }, NXT_CONF_VLDT_END }; @@ -263,50 +302,21 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { }; -static nxt_conf_vldt_object_t nxt_conf_vldt_setting_members[] = { - { nxt_string("http"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_http_members }, - - NXT_CONF_VLDT_END -}; - - -static nxt_conf_vldt_object_t nxt_conf_vldt_root_members[] = { - { nxt_string("settings"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_setting_members }, - - { nxt_string("listeners"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_listener }, - - { nxt_string("routes"), - NXT_CONF_VLDT_ARRAY | NXT_CONF_VLDT_OBJECT, +static nxt_conf_vldt_object_t nxt_conf_vldt_websocket_members[] = { + { nxt_string("read_timeout"), + NXT_CONF_VLDT_INTEGER, 0, - &nxt_conf_vldt_routes, + NULL, NULL }, - { nxt_string("applications"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_app }, - - { nxt_string("upstreams"), - NXT_CONF_VLDT_OBJECT, + { nxt_string("keepalive_interval"), + NXT_CONF_VLDT_INTEGER, 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_upstream }, + NULL, + NULL }, - { nxt_string("access_log"), - NXT_CONF_VLDT_STRING, + { nxt_string("max_frame_size"), + NXT_CONF_VLDT_INTEGER, 0, NULL, NULL }, @@ -315,20 +325,16 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_root_members[] = { }; -#if (NXT_TLS) - -static nxt_conf_vldt_object_t nxt_conf_vldt_tls_members[] = { - { nxt_string("certificate"), - NXT_CONF_VLDT_STRING, +static nxt_conf_vldt_object_t nxt_conf_vldt_static_members[] = { + { nxt_string("mime_types"), + NXT_CONF_VLDT_OBJECT, 0, - &nxt_conf_vldt_certificate, + &nxt_conf_vldt_mtypes, NULL }, NXT_CONF_VLDT_END }; -#endif - static nxt_conf_vldt_object_t nxt_conf_vldt_listener_members[] = { { nxt_string("pass"), @@ -357,6 +363,38 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_listener_members[] = { }; +#if (NXT_TLS) + +static nxt_conf_vldt_object_t nxt_conf_vldt_tls_members[] = { + { nxt_string("certificate"), + NXT_CONF_VLDT_STRING, + 0, + &nxt_conf_vldt_certificate, + NULL }, + + NXT_CONF_VLDT_END +}; + +#endif + + +static nxt_conf_vldt_object_t nxt_conf_vldt_route_members[] = { + { nxt_string("match"), + NXT_CONF_VLDT_OBJECT, + 0, + &nxt_conf_vldt_object, + (void *) &nxt_conf_vldt_match_members }, + + { nxt_string("action"), + NXT_CONF_VLDT_OBJECT, + 0, + &nxt_conf_vldt_action, + NULL }, + + NXT_CONF_VLDT_END +}; + + static nxt_conf_vldt_object_t nxt_conf_vldt_match_members[] = { { nxt_string("method"), NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, @@ -472,215 +510,209 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_proxy_action_members[] = { }; -static nxt_conf_vldt_object_t nxt_conf_vldt_route_members[] = { - { nxt_string("match"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_match_members }, +static nxt_conf_vldt_object_t nxt_conf_vldt_external_members[] = { + { nxt_string("executable"), + NXT_CONF_VLDT_STRING, + NXT_CONF_VLDT_REQUIRED, + NULL, + NULL }, - { nxt_string("action"), - NXT_CONF_VLDT_OBJECT, + { nxt_string("arguments"), + NXT_CONF_VLDT_ARRAY, 0, - &nxt_conf_vldt_action, - NULL }, + &nxt_conf_vldt_array_iterator, + (void *) &nxt_conf_vldt_argument }, - NXT_CONF_VLDT_END + NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) }; -static nxt_conf_vldt_object_t nxt_conf_vldt_app_limits_members[] = { - { nxt_string("timeout"), - NXT_CONF_VLDT_INTEGER, +static nxt_conf_vldt_object_t nxt_conf_vldt_python_members[] = { + { nxt_string("home"), + NXT_CONF_VLDT_STRING, 0, NULL, NULL }, - { nxt_string("requests"), - NXT_CONF_VLDT_INTEGER, + { nxt_string("path"), + NXT_CONF_VLDT_STRING, 0, NULL, NULL }, - { nxt_string("shm"), - NXT_CONF_VLDT_INTEGER, + { nxt_string("module"), + NXT_CONF_VLDT_STRING, + NXT_CONF_VLDT_REQUIRED, + NULL, + NULL }, + + { nxt_string("callable"), + NXT_CONF_VLDT_STRING, 0, NULL, NULL }, - NXT_CONF_VLDT_END + NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) }; -static nxt_conf_vldt_object_t nxt_conf_vldt_app_processes_members[] = { - { nxt_string("spare"), - NXT_CONF_VLDT_INTEGER, +static nxt_conf_vldt_object_t nxt_conf_vldt_php_members[] = { + { nxt_string("root"), + NXT_CONF_VLDT_ANY_TYPE, 0, - NULL, - NULL }, + &nxt_conf_vldt_php_targets_exclusive, + (void *) "root" }, - { nxt_string("max"), - NXT_CONF_VLDT_INTEGER, + { nxt_string("script"), + NXT_CONF_VLDT_ANY_TYPE, 0, - NULL, - NULL }, + &nxt_conf_vldt_php_targets_exclusive, + (void *) "script" }, - { nxt_string("idle_timeout"), - NXT_CONF_VLDT_INTEGER, + { nxt_string("index"), + NXT_CONF_VLDT_ANY_TYPE, 0, - NULL, + &nxt_conf_vldt_php_targets_exclusive, + (void *) "index" }, + + { nxt_string("targets"), + NXT_CONF_VLDT_OBJECT, + 0, + &nxt_conf_vldt_php_targets, NULL }, - NXT_CONF_VLDT_END + NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_php_common_members) }; -static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[] = { - -#if (NXT_HAVE_CLONE_NEWUSER) - { nxt_string("credential"), - NXT_CONF_VLDT_BOOLEAN, +static nxt_conf_vldt_object_t nxt_conf_vldt_php_common_members[] = { + { nxt_string("options"), + NXT_CONF_VLDT_OBJECT, 0, - NULL, - NULL }, -#endif + &nxt_conf_vldt_object, + (void *) &nxt_conf_vldt_php_options_members }, -#if (NXT_HAVE_CLONE_NEWPID) - { nxt_string("pid"), - NXT_CONF_VLDT_BOOLEAN, + NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) +}; + + +static nxt_conf_vldt_object_t nxt_conf_vldt_php_options_members[] = { + { nxt_string("file"), + NXT_CONF_VLDT_STRING, 0, NULL, NULL }, -#endif -#if (NXT_HAVE_CLONE_NEWNET) - { nxt_string("network"), - NXT_CONF_VLDT_BOOLEAN, + { nxt_string("admin"), + NXT_CONF_VLDT_OBJECT, 0, - NULL, - NULL }, -#endif + &nxt_conf_vldt_object_iterator, + (void *) &nxt_conf_vldt_php_option }, -#if (NXT_HAVE_CLONE_NEWNS) - { nxt_string("mount"), - NXT_CONF_VLDT_BOOLEAN, + { nxt_string("user"), + NXT_CONF_VLDT_OBJECT, 0, + &nxt_conf_vldt_object_iterator, + (void *) &nxt_conf_vldt_php_option }, + + NXT_CONF_VLDT_END +}; + + +static nxt_conf_vldt_object_t nxt_conf_vldt_php_target_members[] = { + { nxt_string("root"), + NXT_CONF_VLDT_STRING, + NXT_CONF_VLDT_REQUIRED, NULL, NULL }, -#endif -#if (NXT_HAVE_CLONE_NEWUTS) - { nxt_string("uname"), - NXT_CONF_VLDT_BOOLEAN, + { nxt_string("script"), + NXT_CONF_VLDT_STRING, 0, NULL, NULL }, -#endif -#if (NXT_HAVE_CLONE_NEWCGROUP) - { nxt_string("cgroup"), - NXT_CONF_VLDT_BOOLEAN, + { nxt_string("index"), + NXT_CONF_VLDT_STRING, 0, NULL, NULL }, -#endif NXT_CONF_VLDT_END }; -#if (NXT_HAVE_CLONE_NEWUSER) - -static nxt_conf_vldt_object_t nxt_conf_vldt_app_procmap_members[] = { - { nxt_string("container"), - NXT_CONF_VLDT_INTEGER, - 0, +static nxt_conf_vldt_object_t nxt_conf_vldt_php_notargets_members[] = { + { nxt_string("root"), + NXT_CONF_VLDT_STRING, + NXT_CONF_VLDT_REQUIRED, NULL, NULL }, - { nxt_string("host"), - NXT_CONF_VLDT_INTEGER, + { nxt_string("script"), + NXT_CONF_VLDT_STRING, 0, NULL, NULL }, - { nxt_string("size"), - NXT_CONF_VLDT_INTEGER, + { nxt_string("index"), + NXT_CONF_VLDT_STRING, 0, NULL, NULL }, - NXT_CONF_VLDT_END + NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_php_common_members) }; -#endif - -#if (NXT_HAVE_ISOLATION_ROOTFS) - -static nxt_conf_vldt_object_t nxt_conf_vldt_app_automount_members[] = { - { nxt_string("language_deps"), - NXT_CONF_VLDT_BOOLEAN, - 0, +static nxt_conf_vldt_object_t nxt_conf_vldt_perl_members[] = { + { nxt_string("script"), + NXT_CONF_VLDT_STRING, + NXT_CONF_VLDT_REQUIRED, NULL, NULL }, - NXT_CONF_VLDT_END + NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) }; -#endif - -static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[] = { - { nxt_string("namespaces"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_clone_namespaces, - (void *) &nxt_conf_vldt_app_namespaces_members }, +static nxt_conf_vldt_object_t nxt_conf_vldt_ruby_members[] = { + { nxt_string("script"), + NXT_CONF_VLDT_STRING, + NXT_CONF_VLDT_REQUIRED, + NULL, + NULL }, -#if (NXT_HAVE_CLONE_NEWUSER) + NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) +}; - { nxt_string("uidmap"), - NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_clone_uidmap }, - { nxt_string("gidmap"), +static nxt_conf_vldt_object_t nxt_conf_vldt_java_members[] = { + { nxt_string("classpath"), NXT_CONF_VLDT_ARRAY, 0, &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_clone_gidmap }, - -#endif - -#if (NXT_HAVE_ISOLATION_ROOTFS) + (void *) &nxt_conf_vldt_java_classpath }, - { nxt_string("rootfs"), + { nxt_string("webapp"), NXT_CONF_VLDT_STRING, - 0, + NXT_CONF_VLDT_REQUIRED, NULL, NULL }, - { nxt_string("automount"), - NXT_CONF_VLDT_OBJECT, + { nxt_string("options"), + NXT_CONF_VLDT_ARRAY, 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_app_automount_members }, - -#endif - -#if (NXT_HAVE_PR_SET_NO_NEW_PRIVS) + &nxt_conf_vldt_array_iterator, + (void *) &nxt_conf_vldt_java_option }, - { nxt_string("new_privs"), - NXT_CONF_VLDT_BOOLEAN, + { nxt_string("unit_jars"), + NXT_CONF_VLDT_STRING, 0, NULL, NULL }, -#endif - - NXT_CONF_VLDT_END + NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) }; @@ -737,67 +769,44 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_common_members[] = { }; -static nxt_conf_vldt_object_t nxt_conf_vldt_external_members[] = { - { nxt_string("executable"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, - - { nxt_string("arguments"), - NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_argument }, - - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) -}; - - -static nxt_conf_vldt_object_t nxt_conf_vldt_python_members[] = { - { nxt_string("home"), - NXT_CONF_VLDT_STRING, +static nxt_conf_vldt_object_t nxt_conf_vldt_app_limits_members[] = { + { nxt_string("timeout"), + NXT_CONF_VLDT_INTEGER, 0, NULL, NULL }, - { nxt_string("path"), - NXT_CONF_VLDT_STRING, + { nxt_string("requests"), + NXT_CONF_VLDT_INTEGER, 0, NULL, NULL }, - { nxt_string("module"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, - - { nxt_string("callable"), - NXT_CONF_VLDT_STRING, + { nxt_string("shm"), + NXT_CONF_VLDT_INTEGER, 0, NULL, NULL }, - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) + NXT_CONF_VLDT_END }; -static nxt_conf_vldt_object_t nxt_conf_vldt_php_target_members[] = { - { nxt_string("root"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, +static nxt_conf_vldt_object_t nxt_conf_vldt_app_processes_members[] = { + { nxt_string("spare"), + NXT_CONF_VLDT_INTEGER, + 0, NULL, NULL }, - { nxt_string("script"), - NXT_CONF_VLDT_STRING, + { nxt_string("max"), + NXT_CONF_VLDT_INTEGER, 0, NULL, NULL }, - { nxt_string("index"), - NXT_CONF_VLDT_STRING, + { nxt_string("idle_timeout"), + NXT_CONF_VLDT_INTEGER, 0, NULL, NULL }, @@ -806,142 +815,154 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_php_target_members[] = { }; -static nxt_conf_vldt_object_t nxt_conf_vldt_php_options_members[] = { - { nxt_string("file"), - NXT_CONF_VLDT_STRING, +static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[] = { + { nxt_string("namespaces"), + NXT_CONF_VLDT_OBJECT, 0, - NULL, - NULL }, + &nxt_conf_vldt_clone_namespaces, + (void *) &nxt_conf_vldt_app_namespaces_members }, - { nxt_string("admin"), - NXT_CONF_VLDT_OBJECT, +#if (NXT_HAVE_CLONE_NEWUSER) + + { nxt_string("uidmap"), + NXT_CONF_VLDT_ARRAY, 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_php_option }, + &nxt_conf_vldt_array_iterator, + (void *) &nxt_conf_vldt_clone_uidmap }, - { nxt_string("user"), - NXT_CONF_VLDT_OBJECT, + { nxt_string("gidmap"), + NXT_CONF_VLDT_ARRAY, 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_php_option }, + &nxt_conf_vldt_array_iterator, + (void *) &nxt_conf_vldt_clone_gidmap }, - NXT_CONF_VLDT_END -}; +#endif +#if (NXT_HAVE_ISOLATION_ROOTFS) -static nxt_conf_vldt_object_t nxt_conf_vldt_php_common_members[] = { - { nxt_string("options"), + { nxt_string("rootfs"), + NXT_CONF_VLDT_STRING, + 0, + NULL, + NULL }, + + { nxt_string("automount"), NXT_CONF_VLDT_OBJECT, 0, &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_php_options_members }, + (void *) &nxt_conf_vldt_app_automount_members }, - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) -}; +#endif +#if (NXT_HAVE_PR_SET_NO_NEW_PRIVS) -static nxt_conf_vldt_object_t nxt_conf_vldt_php_notargets_members[] = { - { nxt_string("root"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, + { nxt_string("new_privs"), + NXT_CONF_VLDT_BOOLEAN, + 0, NULL, NULL }, - { nxt_string("script"), - NXT_CONF_VLDT_STRING, +#endif + + NXT_CONF_VLDT_END +}; + + +static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[] = { + +#if (NXT_HAVE_CLONE_NEWUSER) + { nxt_string("credential"), + NXT_CONF_VLDT_BOOLEAN, 0, NULL, NULL }, +#endif - { nxt_string("index"), - NXT_CONF_VLDT_STRING, +#if (NXT_HAVE_CLONE_NEWPID) + { nxt_string("pid"), + NXT_CONF_VLDT_BOOLEAN, 0, NULL, NULL }, +#endif - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_php_common_members) -}; - - -static nxt_conf_vldt_object_t nxt_conf_vldt_php_members[] = { - { nxt_string("root"), - NXT_CONF_VLDT_ANY_TYPE, +#if (NXT_HAVE_CLONE_NEWNET) + { nxt_string("network"), + NXT_CONF_VLDT_BOOLEAN, 0, - &nxt_conf_vldt_php_targets_exclusive, - (void *) "root" }, + NULL, + NULL }, +#endif - { nxt_string("script"), - NXT_CONF_VLDT_ANY_TYPE, +#if (NXT_HAVE_CLONE_NEWNS) + { nxt_string("mount"), + NXT_CONF_VLDT_BOOLEAN, 0, - &nxt_conf_vldt_php_targets_exclusive, - (void *) "script" }, + NULL, + NULL }, +#endif - { nxt_string("index"), - NXT_CONF_VLDT_ANY_TYPE, +#if (NXT_HAVE_CLONE_NEWUTS) + { nxt_string("uname"), + NXT_CONF_VLDT_BOOLEAN, 0, - &nxt_conf_vldt_php_targets_exclusive, - (void *) "index" }, + NULL, + NULL }, +#endif - { nxt_string("targets"), - NXT_CONF_VLDT_OBJECT, +#if (NXT_HAVE_CLONE_NEWCGROUP) + { nxt_string("cgroup"), + NXT_CONF_VLDT_BOOLEAN, 0, - &nxt_conf_vldt_php_targets, + NULL, NULL }, +#endif - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_php_common_members) + NXT_CONF_VLDT_END }; -static nxt_conf_vldt_object_t nxt_conf_vldt_perl_members[] = { - { nxt_string("script"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, +#if (NXT_HAVE_ISOLATION_ROOTFS) + +static nxt_conf_vldt_object_t nxt_conf_vldt_app_automount_members[] = { + { nxt_string("language_deps"), + NXT_CONF_VLDT_BOOLEAN, + 0, NULL, NULL }, - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) + NXT_CONF_VLDT_END }; +#endif -static nxt_conf_vldt_object_t nxt_conf_vldt_ruby_members[] = { - { nxt_string("script"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, - - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) -}; +#if (NXT_HAVE_CLONE_NEWUSER) -static nxt_conf_vldt_object_t nxt_conf_vldt_java_members[] = { - { nxt_string("classpath"), - NXT_CONF_VLDT_ARRAY, +static nxt_conf_vldt_object_t nxt_conf_vldt_app_procmap_members[] = { + { nxt_string("container"), + NXT_CONF_VLDT_INTEGER, 0, - &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_java_classpath }, - - { nxt_string("webapp"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, NULL, NULL }, - { nxt_string("options"), - NXT_CONF_VLDT_ARRAY, + { nxt_string("host"), + NXT_CONF_VLDT_INTEGER, 0, - &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_java_option }, + NULL, + NULL }, - { nxt_string("unit_jars"), - NXT_CONF_VLDT_STRING, + { nxt_string("size"), + NXT_CONF_VLDT_INTEGER, 0, NULL, NULL }, - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) + NXT_CONF_VLDT_END }; +#endif + static nxt_conf_vldt_object_t nxt_conf_vldt_upstream_members[] = { { nxt_string("servers"), -- cgit From f541cbcce4f3790551c325755c74ef2cf18b8a2c Mon Sep 17 00:00:00 2001 From: Igor Sysoev Date: Tue, 13 Oct 2020 12:56:56 +0300 Subject: Using C99 style declaration. --- src/nxt_conf_validation.c | 986 ++++++++++++++++++++-------------------------- 1 file changed, 423 insertions(+), 563 deletions(-) diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index c01cee62..fec471b1 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -50,8 +50,8 @@ typedef struct { } nxt_conf_vldt_object_t; -#define NXT_CONF_VLDT_NEXT(f) { nxt_null_string, 0, 0, NULL, (f) } -#define NXT_CONF_VLDT_END { nxt_null_string, 0, 0, NULL, NULL } +#define NXT_CONF_VLDT_NEXT(f) { .data = f } +#define NXT_CONF_VLDT_END { .name = nxt_null_string } typedef nxt_int_t (*nxt_conf_vldt_member_t)(nxt_conf_validation_t *vldt, @@ -192,171 +192,135 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_automount_members[]; static nxt_conf_vldt_object_t nxt_conf_vldt_root_members[] = { - { nxt_string("settings"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_setting_members }, - - { nxt_string("listeners"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_listener }, - - { nxt_string("routes"), - NXT_CONF_VLDT_ARRAY | NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_routes, - NULL }, - - { nxt_string("applications"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_app }, - - { nxt_string("upstreams"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_upstream }, - - { nxt_string("access_log"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, + { + .name = nxt_string("settings"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object, + .data = (void *) &nxt_conf_vldt_setting_members, + }, { + .name = nxt_string("listeners"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object_iterator, + .data = (void *) &nxt_conf_vldt_listener, + }, { + .name = nxt_string("routes"), + .type = NXT_CONF_VLDT_ARRAY | NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_routes, + }, { + .name = nxt_string("applications"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object_iterator, + .data = (void *) &nxt_conf_vldt_app, + }, { + .name = nxt_string("upstreams"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object_iterator, + .data = (void *) &nxt_conf_vldt_upstream, + }, { + .name = nxt_string("access_log"), + .type = NXT_CONF_VLDT_STRING, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_setting_members[] = { - { nxt_string("http"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_http_members }, + { + .name = nxt_string("http"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object, + .data = (void *) &nxt_conf_vldt_http_members, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { - { nxt_string("header_read_timeout"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("body_read_timeout"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("send_timeout"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("idle_timeout"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("body_buffer_size"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("max_body_size"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("body_temp_path"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("websocket"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_websocket_members }, - - { nxt_string("static"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_static_members }, + { + .name = nxt_string("header_read_timeout"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("body_read_timeout"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("send_timeout"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("idle_timeout"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("body_buffer_size"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("max_body_size"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("body_temp_path"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("websocket"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object, + .data = (void *) &nxt_conf_vldt_websocket_members, + }, { + .name = nxt_string("static"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object, + .data = (void *) &nxt_conf_vldt_static_members, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_websocket_members[] = { - { nxt_string("read_timeout"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("keepalive_interval"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("max_frame_size"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, + { + .name = nxt_string("read_timeout"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + + .name = nxt_string("keepalive_interval"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("max_frame_size"), + .type = NXT_CONF_VLDT_INTEGER, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_static_members[] = { - { nxt_string("mime_types"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_mtypes, - NULL }, + { + .name = nxt_string("mime_types"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_mtypes, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_listener_members[] = { - { nxt_string("pass"), - NXT_CONF_VLDT_STRING, - 0, - &nxt_conf_vldt_pass, - NULL }, - - { nxt_string("application"), - NXT_CONF_VLDT_STRING, - 0, - &nxt_conf_vldt_app_name, - NULL }, + { + .name = nxt_string("pass"), + .type = NXT_CONF_VLDT_STRING, + .validator = &nxt_conf_vldt_pass, + }, { + .name = nxt_string("application"), + .type = NXT_CONF_VLDT_STRING, + .validator = &nxt_conf_vldt_app_name, + }, #if (NXT_TLS) - - { nxt_string("tls"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_tls_members }, - + { + .name = nxt_string("tls"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object, + .data = (void *) &nxt_conf_vldt_tls_members, + }, #endif NXT_CONF_VLDT_END @@ -366,11 +330,11 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_listener_members[] = { #if (NXT_TLS) static nxt_conf_vldt_object_t nxt_conf_vldt_tls_members[] = { - { nxt_string("certificate"), - NXT_CONF_VLDT_STRING, - 0, - &nxt_conf_vldt_certificate, - NULL }, + { + .name = nxt_string("certificate"), + .type = NXT_CONF_VLDT_STRING, + .validator = &nxt_conf_vldt_certificate, + }, NXT_CONF_VLDT_END }; @@ -379,489 +343,398 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_tls_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_route_members[] = { - { nxt_string("match"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_match_members }, - - { nxt_string("action"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_action, - NULL }, + { + .name = nxt_string("match"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object, + .data = (void *) &nxt_conf_vldt_match_members, + }, { + .name = nxt_string("action"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_action, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_match_members[] = { - { nxt_string("method"), - NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_patterns, - NULL }, - - { nxt_string("scheme"), - NXT_CONF_VLDT_STRING, - 0, - &nxt_conf_vldt_match_scheme_pattern, - NULL }, - - { nxt_string("host"), - NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_patterns, - NULL }, - - { nxt_string("source"), - NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_addrs, - NULL }, - - { nxt_string("destination"), - NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_addrs, - NULL }, - - { nxt_string("uri"), - NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_encoded_patterns, - NULL }, - - { nxt_string("arguments"), - NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_encoded_patterns_sets, - NULL }, - - { nxt_string("headers"), - NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_patterns_sets, - NULL }, - - { nxt_string("cookies"), - NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_patterns_sets, - NULL }, + { + .name = nxt_string("method"), + .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + .validator = &nxt_conf_vldt_match_patterns, + }, { + .name = nxt_string("scheme"), + .type = NXT_CONF_VLDT_STRING, + .validator = &nxt_conf_vldt_match_scheme_pattern, + }, { + .name = nxt_string("host"), + .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + .validator = &nxt_conf_vldt_match_patterns, + }, { + .name = nxt_string("source"), + .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + .validator = &nxt_conf_vldt_match_addrs, + }, { + .name = nxt_string("destination"), + .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + .validator = &nxt_conf_vldt_match_addrs, + }, { + .name = nxt_string("uri"), + .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + .validator = &nxt_conf_vldt_match_encoded_patterns, + }, { + .name = nxt_string("arguments"), + .type = NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, + .validator = &nxt_conf_vldt_match_encoded_patterns_sets, + }, { + .name = nxt_string("headers"), + .type = NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, + .validator = &nxt_conf_vldt_match_patterns_sets, + }, { + .name = nxt_string("cookies"), + .type = NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, + .validator = &nxt_conf_vldt_match_patterns_sets, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_pass_action_members[] = { - { nxt_string("pass"), - NXT_CONF_VLDT_STRING, - 0, - &nxt_conf_vldt_pass, - NULL }, + { + .name = nxt_string("pass"), + .type = NXT_CONF_VLDT_STRING, + .validator = &nxt_conf_vldt_pass, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_return_action_members[] = { - { nxt_string("return"), - NXT_CONF_VLDT_INTEGER, - 0, - &nxt_conf_vldt_return, - NULL }, - - { nxt_string("location"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, + { + .name = nxt_string("return"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = &nxt_conf_vldt_return, + }, { + .name = nxt_string("location"), + .type = NXT_CONF_VLDT_STRING, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_share_action_members[] = { - { nxt_string("share"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("fallback"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_action, - NULL }, + { + .name = nxt_string("share"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("fallback"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_action, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_proxy_action_members[] = { - { nxt_string("proxy"), - NXT_CONF_VLDT_STRING, - 0, - &nxt_conf_vldt_proxy, - NULL }, + { + .name = nxt_string("proxy"), + .type = NXT_CONF_VLDT_STRING, + .validator = &nxt_conf_vldt_proxy, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_external_members[] = { - { nxt_string("executable"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, - - { nxt_string("arguments"), - NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_argument }, + { + .name = nxt_string("executable"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + }, { + .name = nxt_string("arguments"), + .type = NXT_CONF_VLDT_ARRAY, + .validator = &nxt_conf_vldt_array_iterator, + .data = (void *) &nxt_conf_vldt_argument, + }, NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) }; static nxt_conf_vldt_object_t nxt_conf_vldt_python_members[] = { - { nxt_string("home"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("path"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("module"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, - - { nxt_string("callable"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, + { + .name = nxt_string("home"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("path"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("module"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + }, { + .name = nxt_string("callable"), + .type = NXT_CONF_VLDT_STRING, + }, NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) }; static nxt_conf_vldt_object_t nxt_conf_vldt_php_members[] = { - { nxt_string("root"), - NXT_CONF_VLDT_ANY_TYPE, - 0, - &nxt_conf_vldt_php_targets_exclusive, - (void *) "root" }, - - { nxt_string("script"), - NXT_CONF_VLDT_ANY_TYPE, - 0, - &nxt_conf_vldt_php_targets_exclusive, - (void *) "script" }, - - { nxt_string("index"), - NXT_CONF_VLDT_ANY_TYPE, - 0, - &nxt_conf_vldt_php_targets_exclusive, - (void *) "index" }, - - { nxt_string("targets"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_php_targets, - NULL }, + { + .name = nxt_string("root"), + .type = NXT_CONF_VLDT_ANY_TYPE, + .validator = &nxt_conf_vldt_php_targets_exclusive, + .data = (void *) "root", + }, { + .name = nxt_string("script"), + .type = NXT_CONF_VLDT_ANY_TYPE, + .validator = &nxt_conf_vldt_php_targets_exclusive, + .data = (void *) "script", + }, { + .name = nxt_string("index"), + .type = NXT_CONF_VLDT_ANY_TYPE, + .validator = &nxt_conf_vldt_php_targets_exclusive, + .data = (void *) "index", + }, { + .name = nxt_string("targets"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_php_targets, + }, NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_php_common_members) }; static nxt_conf_vldt_object_t nxt_conf_vldt_php_common_members[] = { - { nxt_string("options"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_php_options_members }, + { + .name = nxt_string("options"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object, + .data = (void *) &nxt_conf_vldt_php_options_members, + }, NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) }; static nxt_conf_vldt_object_t nxt_conf_vldt_php_options_members[] = { - { nxt_string("file"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("admin"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_php_option }, - - { nxt_string("user"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_php_option }, + { + .name = nxt_string("file"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("admin"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object_iterator, + .data = (void *) &nxt_conf_vldt_php_option, + }, { + .name = nxt_string("user"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object_iterator, + .data = (void *) &nxt_conf_vldt_php_option, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_php_target_members[] = { - { nxt_string("root"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, - - { nxt_string("script"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("index"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, + { + .name = nxt_string("root"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + }, { + .name = nxt_string("script"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("index"), + .type = NXT_CONF_VLDT_STRING, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_php_notargets_members[] = { - { nxt_string("root"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, - - { nxt_string("script"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("index"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, + { + .name = nxt_string("root"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + }, { + .name = nxt_string("script"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("index"), + .type = NXT_CONF_VLDT_STRING, + }, NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_php_common_members) }; static nxt_conf_vldt_object_t nxt_conf_vldt_perl_members[] = { - { nxt_string("script"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, + { + .name = nxt_string("script"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + }, NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) }; static nxt_conf_vldt_object_t nxt_conf_vldt_ruby_members[] = { - { nxt_string("script"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, + { + .name = nxt_string("script"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + }, NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) }; static nxt_conf_vldt_object_t nxt_conf_vldt_java_members[] = { - { nxt_string("classpath"), - NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_java_classpath }, - - { nxt_string("webapp"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, - - { nxt_string("options"), - NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_java_option }, - - { nxt_string("unit_jars"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, + { + .name = nxt_string("classpath"), + .type = NXT_CONF_VLDT_ARRAY, + .validator = &nxt_conf_vldt_array_iterator, + .data = (void *) &nxt_conf_vldt_java_classpath, + }, { + .name = nxt_string("webapp"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + }, { + .name = nxt_string("options"), + .type = NXT_CONF_VLDT_ARRAY, + .validator = &nxt_conf_vldt_array_iterator, + .data = (void *) &nxt_conf_vldt_java_option, + }, { + .name = nxt_string("unit_jars"), + .type = NXT_CONF_VLDT_STRING, + }, NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) }; static nxt_conf_vldt_object_t nxt_conf_vldt_common_members[] = { - { nxt_string("type"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("limits"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_app_limits_members }, - - { nxt_string("processes"), - NXT_CONF_VLDT_INTEGER | NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_processes, - (void *) &nxt_conf_vldt_app_processes_members }, - - { nxt_string("user"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("group"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("working_directory"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("environment"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_environment }, - - { nxt_string("isolation"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_isolation, - (void *) &nxt_conf_vldt_app_isolation_members }, + { + .name = nxt_string("type"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("limits"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object, + .data = (void *) &nxt_conf_vldt_app_limits_members, + }, { + .name = nxt_string("processes"), + .type = NXT_CONF_VLDT_INTEGER | NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_processes, + .data = (void *) &nxt_conf_vldt_app_processes_members, + }, { + .name = nxt_string("user"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("group"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("working_directory"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("environment"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object_iterator, + .data = (void *) &nxt_conf_vldt_environment, + }, { + .name = nxt_string("isolation"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_isolation, + .data = (void *) &nxt_conf_vldt_app_isolation_members, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_app_limits_members[] = { - { nxt_string("timeout"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("requests"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("shm"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, + { + .name = nxt_string("timeout"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("requests"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("shm"), + .type = NXT_CONF_VLDT_INTEGER, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_app_processes_members[] = { - { nxt_string("spare"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("max"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("idle_timeout"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, + { + .name = nxt_string("spare"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("max"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("idle_timeout"), + .type = NXT_CONF_VLDT_INTEGER, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[] = { - { nxt_string("namespaces"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_clone_namespaces, - (void *) &nxt_conf_vldt_app_namespaces_members }, + { + .name = nxt_string("namespaces"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_clone_namespaces, + .data = (void *) &nxt_conf_vldt_app_namespaces_members, + }, #if (NXT_HAVE_CLONE_NEWUSER) - - { nxt_string("uidmap"), - NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_clone_uidmap }, - - { nxt_string("gidmap"), - NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_clone_gidmap }, - + { + .name = nxt_string("uidmap"), + .type = NXT_CONF_VLDT_ARRAY, + .validator = &nxt_conf_vldt_array_iterator, + .data = (void *) &nxt_conf_vldt_clone_uidmap, + }, { + .name = nxt_string("gidmap"), + .type = NXT_CONF_VLDT_ARRAY, + .validator = &nxt_conf_vldt_array_iterator, + .data = (void *) &nxt_conf_vldt_clone_gidmap, + }, #endif #if (NXT_HAVE_ISOLATION_ROOTFS) - - { nxt_string("rootfs"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("automount"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_app_automount_members }, - + { + .name = nxt_string("rootfs"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("automount"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object, + .data = (void *) &nxt_conf_vldt_app_automount_members, + }, #endif #if (NXT_HAVE_PR_SET_NO_NEW_PRIVS) - - { nxt_string("new_privs"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, - + { + .name = nxt_string("new_privs"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, #endif NXT_CONF_VLDT_END @@ -871,51 +744,45 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[] = { #if (NXT_HAVE_CLONE_NEWUSER) - { nxt_string("credential"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, + { + .name = nxt_string("credential"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, #endif #if (NXT_HAVE_CLONE_NEWPID) - { nxt_string("pid"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, + { + .name = nxt_string("pid"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, #endif #if (NXT_HAVE_CLONE_NEWNET) - { nxt_string("network"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, + { + .name = nxt_string("network"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, #endif #if (NXT_HAVE_CLONE_NEWNS) - { nxt_string("mount"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, + { + .name = nxt_string("mount"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, #endif #if (NXT_HAVE_CLONE_NEWUTS) - { nxt_string("uname"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, + { + .name = nxt_string("uname"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, #endif #if (NXT_HAVE_CLONE_NEWCGROUP) - { nxt_string("cgroup"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, + { + .name = nxt_string("cgroup"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, #endif NXT_CONF_VLDT_END @@ -925,11 +792,10 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[] = { #if (NXT_HAVE_ISOLATION_ROOTFS) static nxt_conf_vldt_object_t nxt_conf_vldt_app_automount_members[] = { - { nxt_string("language_deps"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, + { + .name = nxt_string("language_deps"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, NXT_CONF_VLDT_END }; @@ -940,23 +806,16 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_automount_members[] = { #if (NXT_HAVE_CLONE_NEWUSER) static nxt_conf_vldt_object_t nxt_conf_vldt_app_procmap_members[] = { - { nxt_string("container"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("host"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("size"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, + { + .name = nxt_string("container"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("host"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("size"), + .type = NXT_CONF_VLDT_INTEGER, + }, NXT_CONF_VLDT_END }; @@ -965,22 +824,23 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_procmap_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_upstream_members[] = { - { nxt_string("servers"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_server }, + { + .name = nxt_string("servers"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = &nxt_conf_vldt_object_iterator, + .data = (void *) &nxt_conf_vldt_server, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_upstream_server_members[] = { - { nxt_string("weight"), - NXT_CONF_VLDT_NUMBER, - 0, - &nxt_conf_vldt_server_weight, - NULL }, + { + .name = nxt_string("weight"), + .type = NXT_CONF_VLDT_NUMBER, + .validator = &nxt_conf_vldt_server_weight, + }, NXT_CONF_VLDT_END }; -- cgit From 90b2c9f7d62a1f1f9cee11fba0ca47821de9faa7 Mon Sep 17 00:00:00 2001 From: Igor Sysoev Date: Tue, 13 Oct 2020 12:56:56 +0300 Subject: Using union instead of "void *". --- src/nxt_conf_validation.c | 212 ++++++++++++++++++++++++---------------------- 1 file changed, 110 insertions(+), 102 deletions(-) diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index fec471b1..b44509e3 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -39,26 +39,34 @@ typedef enum { typedef nxt_int_t (*nxt_conf_vldt_handler_t)(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +typedef nxt_int_t (*nxt_conf_vldt_member_t)(nxt_conf_validation_t *vldt, + nxt_str_t *name, + nxt_conf_value_t *value); +typedef nxt_int_t (*nxt_conf_vldt_element_t)(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value); -typedef struct { - nxt_str_t name; - nxt_conf_vldt_type_t type:32; - nxt_conf_vldt_flags_t flags:32; - nxt_conf_vldt_handler_t validator; - void *data; -} nxt_conf_vldt_object_t; +typedef struct nxt_conf_vldt_object_s nxt_conf_vldt_object_t; +struct nxt_conf_vldt_object_s { + nxt_str_t name; + nxt_conf_vldt_type_t type:32; + nxt_conf_vldt_flags_t flags:32; + nxt_conf_vldt_handler_t validator; -#define NXT_CONF_VLDT_NEXT(f) { .data = f } -#define NXT_CONF_VLDT_END { .name = nxt_null_string } + union { + nxt_conf_vldt_object_t *members; + nxt_conf_vldt_object_t *next; + nxt_conf_vldt_member_t object; + nxt_conf_vldt_element_t array; + const char *string; + } u; +}; -typedef nxt_int_t (*nxt_conf_vldt_member_t)(nxt_conf_validation_t *vldt, - nxt_str_t *name, - nxt_conf_value_t *value); -typedef nxt_int_t (*nxt_conf_vldt_element_t)(nxt_conf_validation_t *vldt, - nxt_conf_value_t *value); +#define NXT_CONF_VLDT_NEXT(next) { .u.members = next } +#define NXT_CONF_VLDT_END { .name = nxt_null_string } + static nxt_int_t nxt_conf_vldt_type(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_conf_value_t *value, nxt_conf_vldt_type_t type); @@ -195,27 +203,27 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_root_members[] = { { .name = nxt_string("settings"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object, - .data = (void *) &nxt_conf_vldt_setting_members, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_setting_members, }, { .name = nxt_string("listeners"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object_iterator, - .data = (void *) &nxt_conf_vldt_listener, + .validator = nxt_conf_vldt_object_iterator, + .u.object = nxt_conf_vldt_listener, }, { .name = nxt_string("routes"), .type = NXT_CONF_VLDT_ARRAY | NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_routes, + .validator = nxt_conf_vldt_routes, }, { .name = nxt_string("applications"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object_iterator, - .data = (void *) &nxt_conf_vldt_app, + .validator = nxt_conf_vldt_object_iterator, + .u.object = nxt_conf_vldt_app, }, { .name = nxt_string("upstreams"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object_iterator, - .data = (void *) &nxt_conf_vldt_upstream, + .validator = nxt_conf_vldt_object_iterator, + .u.object = nxt_conf_vldt_upstream, }, { .name = nxt_string("access_log"), .type = NXT_CONF_VLDT_STRING, @@ -229,8 +237,8 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_setting_members[] = { { .name = nxt_string("http"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object, - .data = (void *) &nxt_conf_vldt_http_members, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_http_members, }, NXT_CONF_VLDT_END @@ -262,13 +270,13 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { }, { .name = nxt_string("websocket"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object, - .data = (void *) &nxt_conf_vldt_websocket_members, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_websocket_members, }, { .name = nxt_string("static"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object, - .data = (void *) &nxt_conf_vldt_static_members, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_static_members, }, NXT_CONF_VLDT_END @@ -296,7 +304,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_static_members[] = { { .name = nxt_string("mime_types"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_mtypes, + .validator = nxt_conf_vldt_mtypes, }, NXT_CONF_VLDT_END @@ -307,19 +315,19 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_listener_members[] = { { .name = nxt_string("pass"), .type = NXT_CONF_VLDT_STRING, - .validator = &nxt_conf_vldt_pass, + .validator = nxt_conf_vldt_pass, }, { .name = nxt_string("application"), .type = NXT_CONF_VLDT_STRING, - .validator = &nxt_conf_vldt_app_name, + .validator = nxt_conf_vldt_app_name, }, #if (NXT_TLS) { .name = nxt_string("tls"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object, - .data = (void *) &nxt_conf_vldt_tls_members, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_tls_members, }, #endif @@ -333,7 +341,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_tls_members[] = { { .name = nxt_string("certificate"), .type = NXT_CONF_VLDT_STRING, - .validator = &nxt_conf_vldt_certificate, + .validator = nxt_conf_vldt_certificate, }, NXT_CONF_VLDT_END @@ -346,12 +354,12 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_route_members[] = { { .name = nxt_string("match"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object, - .data = (void *) &nxt_conf_vldt_match_members, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_match_members, }, { .name = nxt_string("action"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_action, + .validator = nxt_conf_vldt_action, }, NXT_CONF_VLDT_END @@ -362,39 +370,39 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_match_members[] = { { .name = nxt_string("method"), .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - .validator = &nxt_conf_vldt_match_patterns, + .validator = nxt_conf_vldt_match_patterns, }, { .name = nxt_string("scheme"), .type = NXT_CONF_VLDT_STRING, - .validator = &nxt_conf_vldt_match_scheme_pattern, + .validator = nxt_conf_vldt_match_scheme_pattern, }, { .name = nxt_string("host"), .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - .validator = &nxt_conf_vldt_match_patterns, + .validator = nxt_conf_vldt_match_patterns, }, { .name = nxt_string("source"), .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - .validator = &nxt_conf_vldt_match_addrs, + .validator = nxt_conf_vldt_match_addrs, }, { .name = nxt_string("destination"), .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - .validator = &nxt_conf_vldt_match_addrs, + .validator = nxt_conf_vldt_match_addrs, }, { .name = nxt_string("uri"), .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - .validator = &nxt_conf_vldt_match_encoded_patterns, + .validator = nxt_conf_vldt_match_encoded_patterns, }, { .name = nxt_string("arguments"), .type = NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, - .validator = &nxt_conf_vldt_match_encoded_patterns_sets, + .validator = nxt_conf_vldt_match_encoded_patterns_sets, }, { .name = nxt_string("headers"), .type = NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, - .validator = &nxt_conf_vldt_match_patterns_sets, + .validator = nxt_conf_vldt_match_patterns_sets, }, { .name = nxt_string("cookies"), .type = NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, - .validator = &nxt_conf_vldt_match_patterns_sets, + .validator = nxt_conf_vldt_match_patterns_sets, }, NXT_CONF_VLDT_END @@ -405,7 +413,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_pass_action_members[] = { { .name = nxt_string("pass"), .type = NXT_CONF_VLDT_STRING, - .validator = &nxt_conf_vldt_pass, + .validator = nxt_conf_vldt_pass, }, NXT_CONF_VLDT_END @@ -416,7 +424,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_return_action_members[] = { { .name = nxt_string("return"), .type = NXT_CONF_VLDT_INTEGER, - .validator = &nxt_conf_vldt_return, + .validator = nxt_conf_vldt_return, }, { .name = nxt_string("location"), .type = NXT_CONF_VLDT_STRING, @@ -433,7 +441,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_share_action_members[] = { }, { .name = nxt_string("fallback"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_action, + .validator = nxt_conf_vldt_action, }, NXT_CONF_VLDT_END @@ -444,7 +452,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_proxy_action_members[] = { { .name = nxt_string("proxy"), .type = NXT_CONF_VLDT_STRING, - .validator = &nxt_conf_vldt_proxy, + .validator = nxt_conf_vldt_proxy, }, NXT_CONF_VLDT_END @@ -459,11 +467,11 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_external_members[] = { }, { .name = nxt_string("arguments"), .type = NXT_CONF_VLDT_ARRAY, - .validator = &nxt_conf_vldt_array_iterator, - .data = (void *) &nxt_conf_vldt_argument, + .validator = nxt_conf_vldt_array_iterator, + .u.array = nxt_conf_vldt_argument, }, - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) }; @@ -483,7 +491,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_python_members[] = { .type = NXT_CONF_VLDT_STRING, }, - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) }; @@ -491,25 +499,25 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_php_members[] = { { .name = nxt_string("root"), .type = NXT_CONF_VLDT_ANY_TYPE, - .validator = &nxt_conf_vldt_php_targets_exclusive, - .data = (void *) "root", + .validator = nxt_conf_vldt_php_targets_exclusive, + .u.string = "root", }, { .name = nxt_string("script"), .type = NXT_CONF_VLDT_ANY_TYPE, - .validator = &nxt_conf_vldt_php_targets_exclusive, - .data = (void *) "script", + .validator = nxt_conf_vldt_php_targets_exclusive, + .u.string = "script", }, { .name = nxt_string("index"), .type = NXT_CONF_VLDT_ANY_TYPE, - .validator = &nxt_conf_vldt_php_targets_exclusive, - .data = (void *) "index", + .validator = nxt_conf_vldt_php_targets_exclusive, + .u.string = "index", }, { .name = nxt_string("targets"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_php_targets, + .validator = nxt_conf_vldt_php_targets, }, - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_php_common_members) + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_php_common_members) }; @@ -517,11 +525,11 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_php_common_members[] = { { .name = nxt_string("options"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object, - .data = (void *) &nxt_conf_vldt_php_options_members, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_php_options_members, }, - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) }; @@ -532,13 +540,13 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_php_options_members[] = { }, { .name = nxt_string("admin"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object_iterator, - .data = (void *) &nxt_conf_vldt_php_option, + .validator = nxt_conf_vldt_object_iterator, + .u.object = nxt_conf_vldt_php_option, }, { .name = nxt_string("user"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object_iterator, - .data = (void *) &nxt_conf_vldt_php_option, + .validator = nxt_conf_vldt_object_iterator, + .u.object = nxt_conf_vldt_php_option, }, NXT_CONF_VLDT_END @@ -575,7 +583,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_php_notargets_members[] = { .type = NXT_CONF_VLDT_STRING, }, - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_php_common_members) + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_php_common_members) }; @@ -586,7 +594,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_perl_members[] = { .flags = NXT_CONF_VLDT_REQUIRED, }, - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) }; @@ -597,7 +605,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_ruby_members[] = { .flags = NXT_CONF_VLDT_REQUIRED, }, - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) }; @@ -605,8 +613,8 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_java_members[] = { { .name = nxt_string("classpath"), .type = NXT_CONF_VLDT_ARRAY, - .validator = &nxt_conf_vldt_array_iterator, - .data = (void *) &nxt_conf_vldt_java_classpath, + .validator = nxt_conf_vldt_array_iterator, + .u.array = nxt_conf_vldt_java_classpath, }, { .name = nxt_string("webapp"), .type = NXT_CONF_VLDT_STRING, @@ -614,14 +622,14 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_java_members[] = { }, { .name = nxt_string("options"), .type = NXT_CONF_VLDT_ARRAY, - .validator = &nxt_conf_vldt_array_iterator, - .data = (void *) &nxt_conf_vldt_java_option, + .validator = nxt_conf_vldt_array_iterator, + .u.array = nxt_conf_vldt_java_option, }, { .name = nxt_string("unit_jars"), .type = NXT_CONF_VLDT_STRING, }, - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) }; @@ -632,13 +640,13 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_common_members[] = { }, { .name = nxt_string("limits"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object, - .data = (void *) &nxt_conf_vldt_app_limits_members, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_app_limits_members, }, { .name = nxt_string("processes"), .type = NXT_CONF_VLDT_INTEGER | NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_processes, - .data = (void *) &nxt_conf_vldt_app_processes_members, + .validator = nxt_conf_vldt_processes, + .u.members = nxt_conf_vldt_app_processes_members, }, { .name = nxt_string("user"), .type = NXT_CONF_VLDT_STRING, @@ -651,13 +659,13 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_common_members[] = { }, { .name = nxt_string("environment"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object_iterator, - .data = (void *) &nxt_conf_vldt_environment, + .validator = nxt_conf_vldt_object_iterator, + .u.object = nxt_conf_vldt_environment, }, { .name = nxt_string("isolation"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_isolation, - .data = (void *) &nxt_conf_vldt_app_isolation_members, + .validator = nxt_conf_vldt_isolation, + .u.members = nxt_conf_vldt_app_isolation_members, }, NXT_CONF_VLDT_END @@ -700,21 +708,21 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[] = { { .name = nxt_string("namespaces"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_clone_namespaces, - .data = (void *) &nxt_conf_vldt_app_namespaces_members, + .validator = nxt_conf_vldt_clone_namespaces, + .u.members = nxt_conf_vldt_app_namespaces_members, }, #if (NXT_HAVE_CLONE_NEWUSER) { .name = nxt_string("uidmap"), .type = NXT_CONF_VLDT_ARRAY, - .validator = &nxt_conf_vldt_array_iterator, - .data = (void *) &nxt_conf_vldt_clone_uidmap, + .validator = nxt_conf_vldt_array_iterator, + .u.array = nxt_conf_vldt_clone_uidmap, }, { .name = nxt_string("gidmap"), .type = NXT_CONF_VLDT_ARRAY, - .validator = &nxt_conf_vldt_array_iterator, - .data = (void *) &nxt_conf_vldt_clone_gidmap, + .validator = nxt_conf_vldt_array_iterator, + .u.array = nxt_conf_vldt_clone_gidmap, }, #endif @@ -725,8 +733,8 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[] = { }, { .name = nxt_string("automount"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object, - .data = (void *) &nxt_conf_vldt_app_automount_members, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_app_automount_members, }, #endif @@ -827,8 +835,8 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_upstream_members[] = { { .name = nxt_string("servers"), .type = NXT_CONF_VLDT_OBJECT, - .validator = &nxt_conf_vldt_object_iterator, - .data = (void *) &nxt_conf_vldt_server, + .validator = nxt_conf_vldt_object_iterator, + .u.object = nxt_conf_vldt_server, }, NXT_CONF_VLDT_END @@ -839,7 +847,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_upstream_server_members[] = { { .name = nxt_string("weight"), .type = NXT_CONF_VLDT_NUMBER, - .validator = &nxt_conf_vldt_server_weight, + .validator = nxt_conf_vldt_server_weight, }, NXT_CONF_VLDT_END @@ -1768,8 +1776,8 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, for ( ;; ) { if (vals->name.length == 0) { - if (vals->data != NULL) { - vals = vals->data; + if (vals->u.members != NULL) { + vals = vals->u.members; continue; } @@ -1802,8 +1810,8 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, for ( ;; ) { if (vals->name.length == 0) { - if (vals->data != NULL) { - vals = vals->data; + if (vals->u.members != NULL) { + vals = vals->u.members; continue; } @@ -1823,7 +1831,7 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } if (vals->validator != NULL) { - ret = vals->validator(vldt, member, vals->data); + ret = vals->validator(vldt, member, vals->u.members); if (ret != NXT_OK) { return ret; -- cgit From 9dcb7ec4b791f542a81e552dbb7a1cfa43430dd4 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 14 Oct 2020 16:18:34 +0300 Subject: Java: response locale methods implemented. This closes #479 issue on GitHub. --- src/java/nginx/unit/Response.java | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/java/nginx/unit/Response.java b/src/java/nginx/unit/Response.java index 099d7f15..268b359d 100644 --- a/src/java/nginx/unit/Response.java +++ b/src/java/nginx/unit/Response.java @@ -40,11 +40,13 @@ public class Response implements HttpServletResponse { private String characterEncoding = defaultCharacterEncoding; private String contentType = null; private String contentTypeHeader = null; + private Locale locale = null; private static final Charset ISO_8859_1 = StandardCharsets.ISO_8859_1; private static final Charset UTF_8 = StandardCharsets.UTF_8; private static final String CONTENT_TYPE = "Content-Type"; + private static final byte[] CONTENT_LANGUAGE_BYTES = "Content-Language".getBytes(ISO_8859_1); private static final byte[] SET_COOKIE_BYTES = "Set-Cookie".getBytes(ISO_8859_1); private static final byte[] EXPIRES_BYTES = "Expires".getBytes(ISO_8859_1); @@ -590,9 +592,13 @@ public class Response implements HttpServletResponse { @Override public Locale getLocale() { - log("getLocale"); + trace("getLocale"); - return null; + if (locale == null) { + return Locale.getDefault(); + } + + return locale; } @Override @@ -795,7 +801,16 @@ public class Response implements HttpServletResponse { @Override public void setLocale(Locale loc) { - log("setLocale: " + loc); + trace("setLocale: " + loc); + + if (loc == null || isCommitted()) { + return; + } + + locale = loc; + String lang = locale.toString().replace('_', '-'); + + setHeader(req_info_ptr, CONTENT_LANGUAGE_BYTES, lang.getBytes(ISO_8859_1)); } private void log(String msg) -- cgit From d8628a43d0705deeb3473faf0252288038defc2b Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 14 Oct 2020 18:41:31 +0300 Subject: Fixing uninitialized ncpu value on unsupported platforms. Thanks to @geyslan. This closes #455 issue on GitHub. --- src/nxt_lib.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nxt_lib.c b/src/nxt_lib.c index 1634a2b8..aba07dda 100644 --- a/src/nxt_lib.c +++ b/src/nxt_lib.c @@ -91,9 +91,12 @@ nxt_lib_start(const char *app, char **argv, char ***envp) #elif (NXT_HPUX) n = mpctl(MPC_GETNUMSPUS, NULL, NULL); +#else + n = 0; + #endif - nxt_debug(&nxt_main_task, "ncpu: %ui", n); + nxt_debug(&nxt_main_task, "ncpu: %d", n); if (n > 1) { nxt_ncpu = n; -- cgit From 54837759f36eddb80af22c8d73e103a948221dc7 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Mon, 19 Oct 2020 22:25:29 +0100 Subject: Tests: fixed unit.log print. --- test/conftest.py | 137 ++++++++++++++++++++++---- test/pytest.ini | 2 +- test/test_access_log.py | 40 ++++---- test/test_asgi_application.py | 3 +- test/test_asgi_lifespan.py | 3 +- test/test_asgi_websockets.py | 2 - test/test_go_isolation.py | 40 +++++--- test/test_go_isolation_rootfs.py | 4 +- test/test_java_application.py | 14 +-- test/test_java_isolation_rootfs.py | 19 ++-- test/test_java_websockets.py | 2 - test/test_node_application.py | 18 ++-- test/test_node_websockets.py | 2 - test/test_perl_application.py | 3 +- test/test_php_application.py | 11 ++- test/test_php_isolation.py | 22 +++-- test/test_proxy.py | 12 +-- test/test_proxy_chunked.py | 9 +- test/test_python_application.py | 16 +-- test/test_python_isolation.py | 33 ++++--- test/test_python_isolation_chroot.py | 6 +- test/test_python_procman.py | 5 +- test/test_respawn.py | 5 +- test/test_return.py | 2 - test/test_routing.py | 12 +-- test/test_ruby_application.py | 13 +-- test/test_ruby_isolation.py | 19 ++-- test/test_settings.py | 4 +- test/test_share_fallback.py | 25 +++-- test/test_static.py | 77 +++++++-------- test/test_tls.py | 67 ++++++------- test/test_upstreams_rr.py | 8 +- test/test_usr1.py | 21 ++-- test/test_variables.py | 2 - test/unit/applications/lang/go.py | 10 +- test/unit/applications/lang/java.py | 2 +- test/unit/applications/lang/node.py | 8 +- test/unit/applications/lang/python.py | 1 - test/unit/applications/proto.py | 2 +- test/unit/applications/tls.py | 14 ++- test/unit/control.py | 3 +- test/unit/feature/isolation.py | 20 ++-- test/unit/http.py | 18 ---- test/unit/main.py | 179 ++-------------------------------- 44 files changed, 427 insertions(+), 488 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index b62264ca..0b9adf69 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -4,11 +4,13 @@ import platform import re import shutil import signal +import socket import stat import subprocess import sys import tempfile import time +from multiprocessing import Process import pytest @@ -45,6 +47,7 @@ def pytest_addoption(parser): unit_instance = {} +_processes = [] option = None @@ -66,6 +69,10 @@ def pytest_configure(config): fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, 0) +def skip_alert(*alerts): + option.skip_alerts.extend(alerts) + + def pytest_generate_tests(metafunc): cls = metafunc.cls if not hasattr(cls, 'application_type'): @@ -127,7 +134,7 @@ def pytest_sessionstart(session): break if m is None: - _print_log() + _print_log(log) exit("Unit is writing log too long") # discover available modules from unit.log @@ -154,8 +161,26 @@ def pytest_sessionstart(session): unit_stop() + shutil.rmtree(unit_instance['temp_dir']) + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + # execute all other hooks to obtain the report object + outcome = yield + rep = outcome.get_result() + + # set a report attribute for each phase of a call, which can + # be "setup", "call", "teardown" + + setattr(item, "rep_" + rep.when, rep) + + +@pytest.fixture(autouse=True) +def run(request): + unit = unit_run() + option.temp_dir = unit['temp_dir'] -def setup_method(self): option.skip_alerts = [ r'read signalfd\(4\) failed', r'sendmsg.+failed', @@ -163,6 +188,40 @@ def setup_method(self): ] option.skip_sanitizer = False + yield + + # stop unit + + error = unit_stop() + + if error: + _print_log() + + assert error is None, 'stop unit' + + # stop all processes + + error = stop_processes() + + if error: + _print_log() + + assert error is None, 'stop unit' + + # check unit.log for alerts + + _check_alerts() + + # print unit.log in case of error + + if request.node.rep_call.failed: + _print_log() + + # remove unit.log + + if not option.save_log: + shutil.rmtree(unit['temp_dir']) + def unit_run(): global unit_instance build_dir = option.current_dir + '/build' @@ -204,14 +263,6 @@ def unit_run(): _print_log() exit('Could not start unit') - # dumb (TODO: remove) - option.skip_alerts = [ - r'read signalfd\(4\) failed', - r'sendmsg.+failed', - r'recvmsg.+failed', - ] - option.skip_sanitizer = False - unit_instance['temp_dir'] = temp_dir unit_instance['log'] = temp_dir + '/unit.log' unit_instance['control_sock'] = temp_dir + '/control.unit.sock' @@ -236,8 +287,6 @@ def unit_stop(): p.kill() return 'Could not terminate unit' - shutil.rmtree(unit_instance['temp_dir']) - def public_dir(path): os.chmod(path, 0o777) @@ -267,11 +316,14 @@ def waitforfiles(*files): return ret -def skip_alert(*alerts): - option.skip_alerts.extend(alerts) +def _check_alerts(path=None): + if path is None: + path = unit_instance['log'] + + with open(path, 'r', encoding='utf-8', errors='ignore') as f: + log = f.read() -def _check_alerts(log): found = False alerts = re.findall(r'.+\[alert\].+', log) @@ -286,23 +338,22 @@ def _check_alerts(log): alerts = [al for al in alerts if re.search(skip, al) is None] if alerts: - _print_log(data=log) + _print_log(log) assert not alerts, 'alert(s)' if not option.skip_sanitizer: sanitizer_errors = re.findall('.+Sanitizer.+', log) if sanitizer_errors: - _print_log(data=log) + _print_log(log) assert not sanitizer_errors, 'sanitizer error(s)' if found: print('skipped.') -def _print_log(path=None, data=None): - if path is None: - path = unit_instance['log'] +def _print_log(data=None): + path = unit_instance['log'] print('Path to unit.log:\n' + path + '\n') @@ -317,6 +368,52 @@ def _print_log(path=None, data=None): sys.stdout.write(data) +def run_process(target, *args): + global _processes + + process = Process(target=target, args=args) + process.start() + + _processes.append(process) + +def stop_processes(): + if not _processes: + return + + fail = False + for process in _processes: + if process.is_alive(): + process.terminate() + process.join(timeout=15) + + if process.is_alive(): + fail = True + + if fail: + return 'Fail to stop process(es)' + + +def waitforsocket(port): + ret = False + + for i in range(50): + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('127.0.0.1', port)) + ret = True + break + except: + sock.close() + time.sleep(0.1) + + sock.close() + + assert ret, 'socket connected' + +@pytest.fixture +def temp_dir(request): + return unit_instance['temp_dir'] + @pytest.fixture def is_unsafe(request): return request.config.getoption("--unsafe") diff --git a/test/pytest.ini b/test/pytest.ini index c672788a..0c3a330c 100644 --- a/test/pytest.ini +++ b/test/pytest.ini @@ -1,3 +1,3 @@ [pytest] -addopts = -rs -vvv +addopts = -vvv -s --detailed --print_log python_functions = test_* diff --git a/test/test_access_log.py b/test/test_access_log.py index eaba82ab..511ce6c5 100644 --- a/test/test_access_log.py +++ b/test/test_access_log.py @@ -2,6 +2,8 @@ import time import pytest +from conftest import option +from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython @@ -12,7 +14,7 @@ class TestAccessLog(TestApplicationPython): super().load(script) assert 'success' in self.conf( - '"' + self.temp_dir + '/access.log"', 'access_log' + '"' + option.temp_dir + '/access.log"', 'access_log' ), 'access_log configure' def wait_for_record(self, pattern, name='access.log'): @@ -48,7 +50,7 @@ class TestAccessLog(TestApplicationPython): body='0123456789', ) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"POST / HTTP/1.1" 200 10') is not None @@ -76,7 +78,7 @@ Connection: close raw=True, ) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET / HTTP/1.1" 200 0 "Referer-1" "-"') @@ -98,7 +100,7 @@ Connection: close self.get(sock_type='ipv6') - self.stop() + unit_stop() assert ( self.wait_for_record( @@ -110,7 +112,7 @@ Connection: close def test_access_log_unix(self): self.load('empty') - addr = self.temp_dir + '/sock' + addr = option.temp_dir + '/sock' self.conf( {"unix:" + addr: {"pass": "applications/empty"}}, 'listeners' @@ -118,7 +120,7 @@ Connection: close self.get(sock_type='unix', addr=addr) - self.stop() + unit_stop() assert ( self.wait_for_record( @@ -138,7 +140,7 @@ Connection: close } ) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET / HTTP/1.1" 200 0 "referer-value" "-"') @@ -156,7 +158,7 @@ Connection: close } ) - self.stop() + unit_stop() assert ( self.wait_for_record( @@ -170,7 +172,7 @@ Connection: close self.get(http_10=True) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET / HTTP/1.0" 200 0 "-" "-"') is not None @@ -185,7 +187,7 @@ Connection: close time.sleep(1) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GE" 400 0 "-" "-"') is not None @@ -198,7 +200,7 @@ Connection: close self.http(b"""GET /\n""", raw=True) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET /" 400 \d+ "-" "-"') is not None @@ -213,7 +215,7 @@ Connection: close time.sleep(1) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET /" 400 0 "-" "-"') is not None @@ -228,7 +230,7 @@ Connection: close time.sleep(1) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET / HTTP/1.1" 400 0 "-" "-"') is not None @@ -242,7 +244,7 @@ Connection: close self.get(headers={'Connection': 'close'}) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET / HTTP/1.1" 400 \d+ "-" "-"') @@ -254,7 +256,7 @@ Connection: close self.get(url='/?blah&var=val') - self.stop() + unit_stop() assert ( self.wait_for_record( @@ -270,20 +272,20 @@ Connection: close self.get(url='/delete') - self.stop() + unit_stop() assert self.search_in_log(r'/delete', 'access.log') is None, 'delete' - def test_access_log_change(self): + def test_access_log_change(self, temp_dir): self.load('empty') self.get() - self.conf('"' + self.temp_dir + '/new.log"', 'access_log') + self.conf('"' + option.temp_dir + '/new.log"', 'access_log') self.get() - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET / HTTP/1.1" 200 0 "-" "-"', 'new.log') diff --git a/test/test_asgi_application.py b/test/test_asgi_application.py index 948d9823..1a654ebf 100644 --- a/test/test_asgi_application.py +++ b/test/test_asgi_application.py @@ -4,6 +4,7 @@ from distutils.version import LooseVersion import pytest +from conftest import option from conftest import skip_alert from unit.applications.lang.python import TestApplicationPython @@ -14,7 +15,7 @@ class TestASGIApplication(TestApplicationPython): load_module = 'asgi' def findall(self, pattern): - with open(self.temp_dir + '/unit.log', 'r', errors='ignore') as f: + with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f: return re.findall(pattern, f.read()) def test_asgi_application_variables(self): diff --git a/test/test_asgi_lifespan.py b/test/test_asgi_lifespan.py index c37a1aae..3f29c7e7 100644 --- a/test/test_asgi_lifespan.py +++ b/test/test_asgi_lifespan.py @@ -5,6 +5,7 @@ import pytest from conftest import option from conftest import public_dir +from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython @@ -34,7 +35,7 @@ class TestASGILifespan(TestApplicationPython): assert self.get()['status'] == 204 - self.stop() + unit_stop() is_startup = os.path.isfile(startup_path) is_shutdown = os.path.isfile(shutdown_path) diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py index ab49b130..54984526 100644 --- a/test/test_asgi_websockets.py +++ b/test/test_asgi_websockets.py @@ -18,8 +18,6 @@ class TestASGIWebsockets(TestApplicationPython): ws = TestApplicationWebsocket() def setup_method(self): - super().setup_method() - assert 'success' in self.conf( {'http': {'websocket': {'keepalive_interval': 0}}}, 'settings' ), 'clear keepalive_interval' diff --git a/test/test_go_isolation.py b/test/test_go_isolation.py index 1e7243f6..9a84aa25 100644 --- a/test/test_go_isolation.py +++ b/test/test_go_isolation.py @@ -1,9 +1,13 @@ import grp import os import pwd +import shutil import pytest +from conftest import option +from conftest import unit_run +from conftest import unit_stop from unit.applications.lang.go import TestApplicationGo from unit.feature.isolation import TestFeatureIsolation @@ -14,11 +18,17 @@ class TestGoIsolation(TestApplicationGo): @classmethod def setup_class(cls, complete_check=True): - unit = super().setup_class(complete_check=False) + check = super().setup_class(complete_check=False) - TestFeatureIsolation().check(cls.available, unit.temp_dir) + unit = unit_run() + option.temp_dir = unit['temp_dir'] - return unit if not complete_check else unit.complete() + TestFeatureIsolation().check(option.available, unit['temp_dir']) + + assert unit_stop() is None + shutil.rmtree(unit['temp_dir']) + + return check if not complete_check else check() def unpriv_creds(self): nobody_uid = pwd.getpwnam('nobody').pw_uid @@ -33,14 +43,14 @@ class TestGoIsolation(TestApplicationGo): return (nobody_uid, nogroup_gid, nogroup) def isolation_key(self, key): - return key in self.available['features']['isolation'].keys() + return key in option.available['features']['isolation'].keys() def test_isolation_values(self): self.load('ns_inspect') obj = self.getjson()['body'] - for ns, ns_value in self.available['features']['isolation'].items(): + for ns, ns_value in option.available['features']['isolation'].items(): if ns.upper() in obj['NS']: assert obj['NS'][ns.upper()] == ns_value, '%s match' % ns @@ -198,7 +208,7 @@ class TestGoIsolation(TestApplicationGo): obj = self.getjson()['body'] # all but user and mnt - allns = list(self.available['features']['isolation'].keys()) + allns = list(option.available['features']['isolation'].keys()) allns.remove('user') allns.remove('mnt') @@ -206,7 +216,7 @@ class TestGoIsolation(TestApplicationGo): if ns.upper() in obj['NS']: assert ( obj['NS'][ns.upper()] - == self.available['features']['isolation'][ns] + == option.available['features']['isolation'][ns] ), ('%s match' % ns) assert obj['NS']['MNT'] != self.isolation.getns('mnt'), 'mnt set' @@ -230,7 +240,7 @@ class TestGoIsolation(TestApplicationGo): def test_isolation_namespace_false(self): self.load('ns_inspect') - allns = list(self.available['features']['isolation'].keys()) + allns = list(option.available['features']['isolation'].keys()) remove_list = ['unprivileged_userns_clone', 'ipc', 'cgroup'] allns = [ns for ns in allns if ns not in remove_list] @@ -256,10 +266,10 @@ class TestGoIsolation(TestApplicationGo): if ns.upper() in obj['NS']: assert ( obj['NS'][ns.upper()] - == self.available['features']['isolation'][ns] + == option.available['features']['isolation'][ns] ), ('%s match' % ns) - def test_go_isolation_rootfs_container(self): + def test_go_isolation_rootfs_container(self, temp_dir): if not self.isolation_key('unprivileged_userns_clone'): pytest.skip('unprivileged clone is not available') @@ -268,7 +278,7 @@ class TestGoIsolation(TestApplicationGo): isolation = { 'namespaces': {'mount': True, 'credential': True}, - 'rootfs': self.temp_dir, + 'rootfs': temp_dir, } self.load('ns_inspect', isolation=isolation) @@ -280,7 +290,7 @@ class TestGoIsolation(TestApplicationGo): obj = self.getjson(url='/?file=/bin/sh')['body'] assert obj['FileExists'] == False, 'file should not exists' - def test_go_isolation_rootfs_container_priv(self, is_su): + def test_go_isolation_rootfs_container_priv(self, is_su, temp_dir): if not is_su: pytest.skip('requires root') @@ -289,7 +299,7 @@ class TestGoIsolation(TestApplicationGo): isolation = { 'namespaces': {'mount': True}, - 'rootfs': self.temp_dir, + 'rootfs': temp_dir, } self.load('ns_inspect', isolation=isolation) @@ -301,7 +311,7 @@ class TestGoIsolation(TestApplicationGo): obj = self.getjson(url='/?file=/bin/sh')['body'] assert obj['FileExists'] == False, 'file should not exists' - def test_go_isolation_rootfs_default_tmpfs(self): + def test_go_isolation_rootfs_default_tmpfs(self, temp_dir): if not self.isolation_key('unprivileged_userns_clone'): pytest.skip('unprivileged clone is not available') @@ -310,7 +320,7 @@ class TestGoIsolation(TestApplicationGo): isolation = { 'namespaces': {'mount': True, 'credential': True}, - 'rootfs': self.temp_dir, + 'rootfs': temp_dir, } self.load('ns_inspect', isolation=isolation) diff --git a/test/test_go_isolation_rootfs.py b/test/test_go_isolation_rootfs.py index d8e177b1..1cc59c67 100644 --- a/test/test_go_isolation_rootfs.py +++ b/test/test_go_isolation_rootfs.py @@ -8,7 +8,7 @@ from unit.applications.lang.go import TestApplicationGo class TestGoIsolationRootfs(TestApplicationGo): prerequisites = {'modules': {'go': 'all'}} - def test_go_isolation_rootfs_chroot(self, is_su): + def test_go_isolation_rootfs_chroot(self, is_su, temp_dir): if not is_su: pytest.skip('requires root') @@ -16,7 +16,7 @@ class TestGoIsolationRootfs(TestApplicationGo): pytest.skip('chroot tests not supported on OSX') isolation = { - 'rootfs': self.temp_dir, + 'rootfs': temp_dir, } self.load('ns_inspect', isolation=isolation) diff --git a/test/test_java_application.py b/test/test_java_application.py index afcdf651..efafa6a1 100644 --- a/test/test_java_application.py +++ b/test/test_java_application.py @@ -11,7 +11,7 @@ from unit.applications.lang.java import TestApplicationJava class TestJavaApplication(TestApplicationJava): prerequisites = {'modules': {'java': 'all'}} - def test_java_conf_error(self): + def test_java_conf_error(self, temp_dir): skip_alert( r'realpath.*failed', r'failed to apply new conf', @@ -25,18 +25,18 @@ class TestJavaApplication(TestApplicationJava): "type": "java", "processes": 1, "working_directory": option.test_dir + "/java/empty", - "webapp": self.temp_dir + "/java", - "unit_jars": self.temp_dir + "/no_such_dir", + "webapp": temp_dir + "/java", + "unit_jars": temp_dir + "/no_such_dir", } }, } ), 'conf error' - def test_java_war(self): + def test_java_war(self, temp_dir): self.load('empty_war') assert 'success' in self.conf( - '"' + self.temp_dir + '/java/empty.war"', + '"' + temp_dir + '/java/empty.war"', '/config/applications/empty_war/webapp', ), 'configure war' @@ -969,11 +969,11 @@ class TestJavaApplication(TestApplicationJava): ), 'set date header' assert headers['X-Get-Date'] == date, 'get date header' - def test_java_application_multipart(self): + def test_java_application_multipart(self, temp_dir): self.load('multipart') reldst = '/uploads' - fulldst = self.temp_dir + reldst + fulldst = temp_dir + reldst os.mkdir(fulldst) public_dir(fulldst) diff --git a/test/test_java_isolation_rootfs.py b/test/test_java_isolation_rootfs.py index f0f04df1..4f9e9834 100644 --- a/test/test_java_isolation_rootfs.py +++ b/test/test_java_isolation_rootfs.py @@ -11,14 +11,12 @@ class TestJavaIsolationRootfs(TestApplicationJava): prerequisites = {'modules': {'java': 'all'}} def setup_method(self, is_su): - super().setup_method() - if not is_su: return - os.makedirs(self.temp_dir + '/jars') - os.makedirs(self.temp_dir + '/tmp') - os.chmod(self.temp_dir + '/tmp', 0o777) + os.makedirs(option.temp_dir + '/jars') + os.makedirs(option.temp_dir + '/tmp') + os.chmod(option.temp_dir + '/tmp', 0o777) try: process = subprocess.Popen( @@ -26,7 +24,7 @@ class TestJavaIsolationRootfs(TestApplicationJava): "mount", "--bind", option.current_dir + "/build", - self.temp_dir + "/jars", + option.temp_dir + "/jars", ], stderr=subprocess.STDOUT, ) @@ -42,7 +40,7 @@ class TestJavaIsolationRootfs(TestApplicationJava): try: process = subprocess.Popen( - ["umount", "--lazy", self.temp_dir + "/jars"], + ["umount", "--lazy", option.temp_dir + "/jars"], stderr=subprocess.STDOUT, ) @@ -51,15 +49,12 @@ class TestJavaIsolationRootfs(TestApplicationJava): except: pytest.fail('Cann\'t run mount process.') - # super teardown must happen after unmount to avoid deletion of /build - super().teardown_method() - - def test_java_isolation_rootfs_chroot_war(self, is_su): + def test_java_isolation_rootfs_chroot_war(self, is_su, temp_dir): if not is_su: pytest.skip('require root') isolation = { - 'rootfs': self.temp_dir, + 'rootfs': temp_dir, } self.load('empty_war', isolation=isolation) diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index 7e6d82e8..7586d4aa 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -15,8 +15,6 @@ class TestJavaWebsockets(TestApplicationJava): ws = TestApplicationWebsocket() def setup_method(self): - super().setup_method() - assert 'success' in self.conf( {'http': {'websocket': {'keepalive_interval': 0}}}, 'settings' ), 'clear keepalive_interval' diff --git a/test/test_node_application.py b/test/test_node_application.py index a0b882f3..c8c3a444 100644 --- a/test/test_node_application.py +++ b/test/test_node_application.py @@ -139,11 +139,11 @@ class TestNodeApplication(TestApplicationNode): assert self.get()['body'] == 'buffer', 'write buffer' - def test_node_application_write_callback(self): + def test_node_application_write_callback(self, temp_dir): self.load('write_callback') assert self.get()['body'] == 'helloworld', 'write callback order' - assert waitforfiles(self.temp_dir + '/node/callback'), 'write callback' + assert waitforfiles(temp_dir + '/node/callback'), 'write callback' def test_node_application_write_before_write_head(self): self.load('write_before_write_head') @@ -222,7 +222,7 @@ class TestNodeApplication(TestApplicationNode): assert 'X-Header' not in headers, 'insensitive' assert 'X-header' not in headers, 'insensitive 2' - def test_node_application_promise_handler(self): + def test_node_application_promise_handler(self, temp_dir): self.load('promise_handler') assert ( @@ -236,7 +236,7 @@ class TestNodeApplication(TestApplicationNode): )['status'] == 200 ), 'promise handler request' - assert waitforfiles(self.temp_dir + '/node/callback'), 'promise handler' + assert waitforfiles(temp_dir + '/node/callback'), 'promise handler' def test_node_application_promise_handler_write_after_end(self): self.load('promise_handler') @@ -254,7 +254,7 @@ class TestNodeApplication(TestApplicationNode): == 200 ), 'promise handler request write after end' - def test_node_application_promise_end(self): + def test_node_application_promise_end(self, temp_dir): self.load('promise_end') assert ( @@ -268,9 +268,9 @@ class TestNodeApplication(TestApplicationNode): )['status'] == 200 ), 'promise end request' - assert waitforfiles(self.temp_dir + '/node/callback'), 'promise end' + assert waitforfiles(temp_dir + '/node/callback'), 'promise end' - def test_node_application_promise_multiple_calls(self): + def test_node_application_promise_multiple_calls(self, temp_dir): self.load('promise_handler') self.post( @@ -283,7 +283,7 @@ class TestNodeApplication(TestApplicationNode): ) assert waitforfiles( - self.temp_dir + '/node/callback1' + temp_dir + '/node/callback1' ), 'promise first call' self.post( @@ -296,7 +296,7 @@ class TestNodeApplication(TestApplicationNode): ) assert waitforfiles( - self.temp_dir + '/node/callback2' + temp_dir + '/node/callback2' ), 'promise second call' @pytest.mark.skip('not yet') diff --git a/test/test_node_websockets.py b/test/test_node_websockets.py index 6a6b7f2d..7b65b5c1 100644 --- a/test/test_node_websockets.py +++ b/test/test_node_websockets.py @@ -15,8 +15,6 @@ class TestNodeWebsockets(TestApplicationNode): ws = TestApplicationWebsocket() def setup_method(self): - super().setup_method() - assert 'success' in self.conf( {'http': {'websocket': {'keepalive_interval': 0}}}, 'settings' ), 'clear keepalive_interval' diff --git a/test/test_perl_application.py b/test/test_perl_application.py index 78e32a43..acd76626 100644 --- a/test/test_perl_application.py +++ b/test/test_perl_application.py @@ -3,6 +3,7 @@ import re import pytest from conftest import skip_alert +from conftest import unit_stop from unit.applications.lang.perl import TestApplicationPerl @@ -119,7 +120,7 @@ class TestPerlApplication(TestApplicationPerl): assert self.get()['body'] == '1', 'errors result' - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+Error in application') diff --git a/test/test_php_application.py b/test/test_php_application.py index 063d3e0c..f1a9e293 100644 --- a/test/test_php_application.py +++ b/test/test_php_application.py @@ -6,6 +6,7 @@ import time import pytest from conftest import option +from conftest import unit_stop from unit.applications.lang.php import TestApplicationPHP class TestPHPApplication(TestApplicationPHP): @@ -444,7 +445,7 @@ class TestPHPApplication(TestApplicationPHP): r'012345', self.get()['body'] ), 'disable_classes before' - def test_php_application_error_log(self): + def test_php_application_error_log(self, temp_dir): self.load('error_log') assert self.get()['status'] == 200, 'status' @@ -453,13 +454,13 @@ class TestPHPApplication(TestApplicationPHP): assert self.get()['status'] == 200, 'status 2' - self.stop() + unit_stop() pattern = r'\d{4}\/\d\d\/\d\d\s\d\d:.+\[notice\].+Error in application' assert self.wait_for_record(pattern) is not None, 'errors print' - with open(self.temp_dir + '/unit.log', 'r', errors='ignore') as f: + with open(temp_dir + '/unit.log', 'r', errors='ignore') as f: errs = re.findall(pattern, f.read()) assert len(errs) == 2, 'error_log count' @@ -507,12 +508,12 @@ class TestPHPApplication(TestApplicationPHP): assert resp['status'] == 200, 'status' assert resp['body'] != '', 'body not empty' - def test_php_application_extension_check(self): + def test_php_application_extension_check(self, temp_dir): self.load('phpinfo') assert self.get(url='/index.wrong')['status'] != 200, 'status' - new_root = self.temp_dir + "/php" + new_root = temp_dir + "/php" os.mkdir(new_root) shutil.copy(option.test_dir + '/php/phpinfo/index.wrong', new_root) diff --git a/test/test_php_isolation.py b/test/test_php_isolation.py index 8ab3419a..ac6942c6 100644 --- a/test/test_php_isolation.py +++ b/test/test_php_isolation.py @@ -1,6 +1,10 @@ +import shutil + import pytest from conftest import option +from conftest import unit_run +from conftest import unit_stop from unit.applications.lang.php import TestApplicationPHP from unit.feature.isolation import TestFeatureIsolation @@ -8,18 +12,22 @@ from unit.feature.isolation import TestFeatureIsolation class TestPHPIsolation(TestApplicationPHP): prerequisites = {'modules': {'php': 'any'}, 'features': ['isolation']} - isolation = TestFeatureIsolation() - @classmethod def setup_class(cls, complete_check=True): - unit = super().setup_class(complete_check=False) + check = super().setup_class(complete_check=False) + + unit = unit_run() + option.temp_dir = unit['temp_dir'] + + TestFeatureIsolation().check(option.available, unit['temp_dir']) - TestFeatureIsolation().check(cls.available, unit.temp_dir) + assert unit_stop() is None + shutil.rmtree(unit['temp_dir']) - return unit if not complete_check else unit.complete() + return check if not complete_check else check() def test_php_isolation_rootfs(self, is_su): - isolation_features = self.available['features']['isolation'].keys() + isolation_features = option.available['features']['isolation'].keys() if 'mnt' not in isolation_features: pytest.skip('requires mnt ns') @@ -48,7 +56,7 @@ class TestPHPIsolation(TestApplicationPHP): assert self.get()['status'] == 200, 'empty rootfs' def test_php_isolation_rootfs_extensions(self, is_su): - isolation_features = self.available['features']['isolation'].keys() + isolation_features = option.available['features']['isolation'].keys() if not is_su: if 'user' not in isolation_features: diff --git a/test/test_proxy.py b/test/test_proxy.py index d02c96a7..be3e93fd 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -5,7 +5,9 @@ import time import pytest from conftest import option +from conftest import run_process from conftest import skip_alert +from conftest import waitforsocket from unit.applications.lang.python import TestApplicationPython @@ -60,10 +62,8 @@ Content-Length: 10 return self.post(*args, http_10=True, **kwargs) def setup_method(self): - super().setup_method() - - self.run_process(self.run_server, self.SERVER_PORT) - self.waitforsocket(self.SERVER_PORT) + run_process(self.run_server, self.SERVER_PORT) + waitforsocket(self.SERVER_PORT) assert 'success' in self.conf( { @@ -346,8 +346,8 @@ Content-Length: 10 assert self.get_http10()['status'] == 200, 'status' - def test_proxy_unix(self): - addr = self.temp_dir + '/sock' + def test_proxy_unix(self, temp_dir): + addr = temp_dir + '/sock' assert 'success' in self.conf( { diff --git a/test/test_proxy_chunked.py b/test/test_proxy_chunked.py index 26023617..ae2228fa 100644 --- a/test/test_proxy_chunked.py +++ b/test/test_proxy_chunked.py @@ -3,6 +3,9 @@ import select import socket import time +from conftest import option +from conftest import run_process +from conftest import waitforsocket from unit.applications.lang.python import TestApplicationPython @@ -82,10 +85,8 @@ class TestProxyChunked(TestApplicationPython): return self.get(*args, http_10=True, **kwargs) def setup_method(self): - super().setup_method() - - self.run_process(self.run_server, self.SERVER_PORT, self.temp_dir) - self.waitforsocket(self.SERVER_PORT) + run_process(self.run_server, self.SERVER_PORT, option.temp_dir) + waitforsocket(self.SERVER_PORT) assert 'success' in self.conf( { diff --git a/test/test_python_application.py b/test/test_python_application.py index 3e27a24c..6ea9f4ff 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -5,7 +5,9 @@ import time import pytest +from conftest import option from conftest import skip_alert +from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython @@ -13,7 +15,7 @@ class TestPythonApplication(TestApplicationPython): prerequisites = {'modules': {'python': 'all'}} def findall(self, pattern): - with open(self.temp_dir + '/unit.log', 'r', errors='ignore') as f: + with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f: return re.findall(pattern, f.read()) def test_python_application_variables(self): @@ -156,7 +158,7 @@ custom-header: BLAH self.conf({"listeners": {}, "applications": {}}) - self.stop() + unit_stop() assert ( self.wait_for_record(r'RuntimeError') is not None @@ -344,7 +346,7 @@ Connection: close self.conf({"listeners": {}, "applications": {}}) - self.stop() + unit_stop() assert self.wait_for_record(r'At exit called\.') is not None, 'atexit' @@ -507,7 +509,7 @@ last line: 987654321 self.get() - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+Error in application\.') @@ -550,7 +552,7 @@ last line: 987654321 self.get() - self.stop() + unit_stop() assert self.wait_for_record(r'Close called\.') is not None, 'close' @@ -559,7 +561,7 @@ last line: 987654321 self.get() - self.stop() + unit_stop() assert ( self.wait_for_record(r'Close called\.') is not None @@ -570,7 +572,7 @@ last line: 987654321 self.get() - self.stop() + unit_stop() assert ( self.wait_for_record( diff --git a/test/test_python_isolation.py b/test/test_python_isolation.py index ac678103..34abd1df 100644 --- a/test/test_python_isolation.py +++ b/test/test_python_isolation.py @@ -1,5 +1,10 @@ +import shutil + import pytest +from conftest import option +from conftest import unit_run +from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython from unit.feature.isolation import TestFeatureIsolation @@ -7,18 +12,22 @@ from unit.feature.isolation import TestFeatureIsolation class TestPythonIsolation(TestApplicationPython): prerequisites = {'modules': {'python': 'any'}, 'features': ['isolation']} - isolation = TestFeatureIsolation() - @classmethod def setup_class(cls, complete_check=True): - unit = super().setup_class(complete_check=False) + check = super().setup_class(complete_check=False) + + unit = unit_run() + option.temp_dir = unit['temp_dir'] + + TestFeatureIsolation().check(option.available, unit['temp_dir']) - TestFeatureIsolation().check(cls.available, unit.temp_dir) + assert unit_stop() is None + shutil.rmtree(unit['temp_dir']) - return unit if not complete_check else unit.complete() + return check if not complete_check else check() - def test_python_isolation_rootfs(self, is_su): - isolation_features = self.available['features']['isolation'].keys() + def test_python_isolation_rootfs(self, is_su, temp_dir): + isolation_features = option.available['features']['isolation'].keys() if 'mnt' not in isolation_features: pytest.skip('requires mnt ns') @@ -32,7 +41,7 @@ class TestPythonIsolation(TestApplicationPython): isolation = { 'namespaces': {'credential': not is_su, 'mount': True}, - 'rootfs': self.temp_dir, + 'rootfs': temp_dir, } self.load('empty', isolation=isolation) @@ -42,7 +51,7 @@ class TestPythonIsolation(TestApplicationPython): self.load('ns_inspect', isolation=isolation) assert ( - self.getjson(url='/?path=' + self.temp_dir)['body']['FileExists'] + self.getjson(url='/?path=' + temp_dir)['body']['FileExists'] == False ), 'temp_dir does not exists in rootfs' @@ -66,8 +75,8 @@ class TestPythonIsolation(TestApplicationPython): ret['body']['FileExists'] == True ), 'application exists in rootfs' - def test_python_isolation_rootfs_no_language_deps(self, is_su): - isolation_features = self.available['features']['isolation'].keys() + def test_python_isolation_rootfs_no_language_deps(self, is_su, temp_dir): + isolation_features = option.available['features']['isolation'].keys() if 'mnt' not in isolation_features: pytest.skip('requires mnt ns') @@ -81,7 +90,7 @@ class TestPythonIsolation(TestApplicationPython): isolation = { 'namespaces': {'credential': not is_su, 'mount': True}, - 'rootfs': self.temp_dir, + 'rootfs': temp_dir, 'automount': {'language_deps': False} } diff --git a/test/test_python_isolation_chroot.py b/test/test_python_isolation_chroot.py index 315fee9f..6d178b2e 100644 --- a/test/test_python_isolation_chroot.py +++ b/test/test_python_isolation_chroot.py @@ -7,12 +7,12 @@ from unit.feature.isolation import TestFeatureIsolation class TestPythonIsolation(TestApplicationPython): prerequisites = {'modules': {'python': 'any'}} - def test_python_isolation_chroot(self, is_su): + def test_python_isolation_chroot(self, is_su, temp_dir): if not is_su: pytest.skip('requires root') isolation = { - 'rootfs': self.temp_dir, + 'rootfs': temp_dir, } self.load('empty', isolation=isolation) @@ -22,7 +22,7 @@ class TestPythonIsolation(TestApplicationPython): self.load('ns_inspect', isolation=isolation) assert ( - self.getjson(url='/?path=' + self.temp_dir)['body']['FileExists'] + self.getjson(url='/?path=' + temp_dir)['body']['FileExists'] == False ), 'temp_dir does not exists in rootfs' diff --git a/test/test_python_procman.py b/test/test_python_procman.py index 8eccae3e..ff914fc8 100644 --- a/test/test_python_procman.py +++ b/test/test_python_procman.py @@ -4,6 +4,7 @@ import time import pytest +from conftest import option from unit.applications.lang.python import TestApplicationPython @@ -11,9 +12,7 @@ class TestPythonProcman(TestApplicationPython): prerequisites = {'modules': {'python': 'any'}} def setup_method(self): - super().setup_method() - - self.app_name = "app-" + self.temp_dir.split('/')[-1] + self.app_name = "app-" + option.temp_dir.split('/')[-1] self.app_proc = 'applications/' + self.app_name + '/processes' self.load('empty', self.app_name) diff --git a/test/test_respawn.py b/test/test_respawn.py index 18b9d535..09a806d4 100644 --- a/test/test_respawn.py +++ b/test/test_respawn.py @@ -2,6 +2,7 @@ import re import subprocess import time +from conftest import option from conftest import skip_alert from unit.applications.lang.python import TestApplicationPython @@ -13,9 +14,7 @@ class TestRespawn(TestApplicationPython): PATTERN_CONTROLLER = 'unit: controller' def setup_method(self): - super().setup_method() - - self.app_name = "app-" + self.temp_dir.split('/')[-1] + self.app_name = "app-" + option.temp_dir.split('/')[-1] self.load('empty', self.app_name) diff --git a/test/test_return.py b/test/test_return.py index 64050022..2f7b7ae4 100644 --- a/test/test_return.py +++ b/test/test_return.py @@ -7,8 +7,6 @@ class TestReturn(TestApplicationProto): prerequisites = {} def setup_method(self): - super().setup_method() - self._load_conf( { "listeners": {"*:7080": {"pass": "routes"}}, diff --git a/test/test_routing.py b/test/test_routing.py index 2b528435..9e0f52dc 100644 --- a/test/test_routing.py +++ b/test/test_routing.py @@ -10,8 +10,6 @@ class TestRouting(TestApplicationProto): prerequisites = {'modules': {'python': 'any'}} def setup_method(self): - super().setup_method() - assert 'success' in self.conf( { "listeners": {"*:7080": {"pass": "routes"}}, @@ -417,7 +415,7 @@ class TestRouting(TestApplicationProto): [{"action": {"pass": "upstreams/blah"}}], 'routes' ), 'route pass upstreams invalid' - def test_routes_action_unique(self): + def test_routes_action_unique(self, temp_dir): assert 'success' in self.conf( { "listeners": { @@ -437,7 +435,7 @@ class TestRouting(TestApplicationProto): ) assert 'error' in self.conf( - {"proxy": "http://127.0.0.1:7081", "share": self.temp_dir}, + {"proxy": "http://127.0.0.1:7081", "share": temp_dir}, 'routes/0/action', ), 'proxy share' assert 'error' in self.conf( @@ -445,7 +443,7 @@ class TestRouting(TestApplicationProto): 'routes/0/action', ), 'proxy pass' assert 'error' in self.conf( - {"share": self.temp_dir, "pass": "applications/app"}, + {"share": temp_dir, "pass": "applications/app"}, 'routes/0/action', ), 'share pass' @@ -1665,8 +1663,8 @@ class TestRouting(TestApplicationProto): assert self.get(sock_type='ipv6')['status'] == 200, '0' assert self.get(port=7081)['status'] == 404, '0 ipv4' - def test_routes_source_unix(self): - addr = self.temp_dir + '/sock' + def test_routes_source_unix(self, temp_dir): + addr = temp_dir + '/sock' assert 'success' in self.conf( {"unix:" + addr: {"pass": "routes"}}, 'listeners' diff --git a/test/test_ruby_application.py b/test/test_ruby_application.py index f84935f8..b765f390 100644 --- a/test/test_ruby_application.py +++ b/test/test_ruby_application.py @@ -3,6 +3,7 @@ import re import pytest from conftest import skip_alert +from conftest import unit_stop from unit.applications.lang.ruby import TestApplicationRuby @@ -175,7 +176,7 @@ class TestRubyApplication(TestApplicationRuby): self.get() - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+Error in application') @@ -187,7 +188,7 @@ class TestRubyApplication(TestApplicationRuby): self.get() - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+1234567890') is not None @@ -198,7 +199,7 @@ class TestRubyApplication(TestApplicationRuby): self.get() - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+Error in application') @@ -215,7 +216,7 @@ class TestRubyApplication(TestApplicationRuby): self.get() - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+1234567890') is not None @@ -228,7 +229,7 @@ class TestRubyApplication(TestApplicationRuby): self.conf({"listeners": {}, "applications": {}}) - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+At exit called\.') is not None @@ -289,7 +290,7 @@ class TestRubyApplication(TestApplicationRuby): assert self.get()['status'] == 500, 'body each error status' - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+Failed to run ruby script') diff --git a/test/test_ruby_isolation.py b/test/test_ruby_isolation.py index 13ca0e16..bf934540 100644 --- a/test/test_ruby_isolation.py +++ b/test/test_ruby_isolation.py @@ -1,7 +1,10 @@ +import shutil import pytest from conftest import option +from conftest import unit_run +from conftest import unit_stop from unit.applications.lang.ruby import TestApplicationRuby from unit.feature.isolation import TestFeatureIsolation @@ -9,18 +12,22 @@ from unit.feature.isolation import TestFeatureIsolation class TestRubyIsolation(TestApplicationRuby): prerequisites = {'modules': {'ruby': 'any'}, 'features': ['isolation']} - isolation = TestFeatureIsolation() - @classmethod def setup_class(cls, complete_check=True): - unit = super().setup_class(complete_check=False) + check = super().setup_class(complete_check=False) + + unit = unit_run() + option.temp_dir = unit['temp_dir'] + + TestFeatureIsolation().check(option.available, unit['temp_dir']) - TestFeatureIsolation().check(cls.available, unit.temp_dir) + assert unit_stop() is None + shutil.rmtree(unit['temp_dir']) - return unit if not complete_check else unit.complete() + return check if not complete_check else check() def test_ruby_isolation_rootfs(self, is_su): - isolation_features = self.available['features']['isolation'].keys() + isolation_features = option.available['features']['isolation'].keys() if 'mnt' not in isolation_features: pytest.skip('requires mnt ns') diff --git a/test/test_settings.py b/test/test_settings.py index b0af6b04..30aa2bff 100644 --- a/test/test_settings.py +++ b/test/test_settings.py @@ -146,14 +146,14 @@ Connection: close assert resp['status'] == 200, 'status body read timeout update' - def test_settings_send_timeout(self): + def test_settings_send_timeout(self, temp_dir): self.load('mirror') data_len = 1048576 self.conf({'http': {'send_timeout': 1}}, 'settings') - addr = self.temp_dir + '/sock' + addr = temp_dir + '/sock' self.conf({"unix:" + addr: {'application': 'mirror'}}, 'listeners') diff --git a/test/test_share_fallback.py b/test/test_share_fallback.py index 391d0836..462da9de 100644 --- a/test/test_share_fallback.py +++ b/test/test_share_fallback.py @@ -1,5 +1,6 @@ import os +from conftest import option from conftest import skip_alert from unit.applications.proto import TestApplicationProto @@ -8,14 +9,12 @@ class TestStatic(TestApplicationProto): prerequisites = {} def setup_method(self): - super().setup_method() - - os.makedirs(self.temp_dir + '/assets/dir') - with open(self.temp_dir + '/assets/index.html', 'w') as index: + os.makedirs(option.temp_dir + '/assets/dir') + with open(option.temp_dir + '/assets/index.html', 'w') as index: index.write('0123456789') - os.makedirs(self.temp_dir + '/assets/403') - os.chmod(self.temp_dir + '/assets/403', 0o000) + os.makedirs(option.temp_dir + '/assets/403') + os.chmod(option.temp_dir + '/assets/403', 0o000) self._load_conf( { @@ -23,15 +22,13 @@ class TestStatic(TestApplicationProto): "*:7080": {"pass": "routes"}, "*:7081": {"pass": "routes"}, }, - "routes": [{"action": {"share": self.temp_dir + "/assets"}}], + "routes": [{"action": {"share": option.temp_dir + "/assets"}}], "applications": {}, } ) def teardown_method(self): - os.chmod(self.temp_dir + '/assets/403', 0o777) - - super().teardown_method() + os.chmod(option.temp_dir + '/assets/403', 0o777) def action_update(self, conf): assert 'success' in self.conf(conf, 'routes/0/action') @@ -46,9 +43,9 @@ class TestStatic(TestApplicationProto): assert resp['status'] == 200, 'bad path fallback status' assert resp['body'] == '', 'bad path fallback' - def test_fallback_valid_path(self): + def test_fallback_valid_path(self, temp_dir): self.action_update( - {"share": self.temp_dir + "/assets", "fallback": {"return": 200}} + {"share": temp_dir + "/assets", "fallback": {"return": 200}} ) resp = self.get() assert resp['status'] == 200, 'fallback status' @@ -79,11 +76,11 @@ class TestStatic(TestApplicationProto): assert resp['status'] == 200, 'fallback nested status' assert resp['body'] == '', 'fallback nested' - def test_fallback_share(self): + def test_fallback_share(self, temp_dir): self.action_update( { "share": "/blah", - "fallback": {"share": self.temp_dir + "/assets"}, + "fallback": {"share": temp_dir + "/assets"}, } ) diff --git a/test/test_static.py b/test/test_static.py index 0b82b4e8..de8b71cc 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -3,6 +3,7 @@ import socket import pytest +from conftest import option from conftest import waitforfiles from unit.applications.proto import TestApplicationProto @@ -11,13 +12,13 @@ class TestStatic(TestApplicationProto): prerequisites = {} def setup_method(self): - super().setup_method() - - os.makedirs(self.temp_dir + '/assets/dir') - with open(self.temp_dir + '/assets/index.html', 'w') as index, open( - self.temp_dir + '/assets/README', 'w' - ) as readme, open(self.temp_dir + '/assets/log.log', 'w') as log, open( - self.temp_dir + '/assets/dir/file', 'w' + os.makedirs(option.temp_dir + '/assets/dir') + with open(option.temp_dir + '/assets/index.html', 'w') as index, open( + option.temp_dir + '/assets/README', 'w' + ) as readme, open( + option.temp_dir + '/assets/log.log', 'w' + ) as log, open( + option.temp_dir + '/assets/dir/file', 'w' ) as file: index.write('0123456789') readme.write('readme') @@ -27,7 +28,7 @@ class TestStatic(TestApplicationProto): self._load_conf( { "listeners": {"*:7080": {"pass": "routes"}}, - "routes": [{"action": {"share": self.temp_dir + "/assets"}}], + "routes": [{"action": {"share": option.temp_dir + "/assets"}}], "settings": { "http": { "static": { @@ -54,9 +55,9 @@ class TestStatic(TestApplicationProto): resp['headers']['Content-Type'] == 'text/html' ), 'index not found 2 Content-Type' - def test_static_large_file(self): + def test_static_large_file(self, temp_dir): file_size = 32 * 1024 * 1024 - with open(self.temp_dir + '/assets/large', 'wb') as f: + with open(temp_dir + '/assets/large', 'wb') as f: f.seek(file_size - 1) f.write(b'\0') @@ -65,14 +66,14 @@ class TestStatic(TestApplicationProto): == file_size ), 'large file' - def test_static_etag(self): + def test_static_etag(self, temp_dir): etag = self.get(url='/')['headers']['ETag'] etag_2 = self.get(url='/README')['headers']['ETag'] assert etag != etag_2, 'different ETag' assert etag == self.get(url='/')['headers']['ETag'], 'same ETag' - with open(self.temp_dir + '/assets/index.html', 'w') as f: + with open(temp_dir + '/assets/index.html', 'w') as f: f.write('blah') assert etag != self.get(url='/')['headers']['ETag'], 'new ETag' @@ -83,22 +84,22 @@ class TestStatic(TestApplicationProto): assert resp['headers']['Location'] == '/dir/', 'redirect Location' assert 'Content-Type' not in resp['headers'], 'redirect Content-Type' - def test_static_space_in_name(self): + def test_static_space_in_name(self, temp_dir): os.rename( - self.temp_dir + '/assets/dir/file', - self.temp_dir + '/assets/dir/fi le', + temp_dir + '/assets/dir/file', + temp_dir + '/assets/dir/fi le', ) - assert waitforfiles(self.temp_dir + '/assets/dir/fi le') + assert waitforfiles(temp_dir + '/assets/dir/fi le') assert self.get(url='/dir/fi le')['body'] == 'blah', 'file name' - os.rename(self.temp_dir + '/assets/dir', self.temp_dir + '/assets/di r') - assert waitforfiles(self.temp_dir + '/assets/di r/fi le') + os.rename(temp_dir + '/assets/dir', temp_dir + '/assets/di r') + assert waitforfiles(temp_dir + '/assets/di r/fi le') assert self.get(url='/di r/fi le')['body'] == 'blah', 'dir name' os.rename( - self.temp_dir + '/assets/di r', self.temp_dir + '/assets/ di r ' + temp_dir + '/assets/di r', temp_dir + '/assets/ di r ' ) - assert waitforfiles(self.temp_dir + '/assets/ di r /fi le') + assert waitforfiles(temp_dir + '/assets/ di r /fi le') assert ( self.get(url='/ di r /fi le')['body'] == 'blah' ), 'dir name enclosing' @@ -121,16 +122,16 @@ class TestStatic(TestApplicationProto): ), 'encoded 2' os.rename( - self.temp_dir + '/assets/ di r /fi le', - self.temp_dir + '/assets/ di r / fi le ', + temp_dir + '/assets/ di r /fi le', + temp_dir + '/assets/ di r / fi le ', ) - assert waitforfiles(self.temp_dir + '/assets/ di r / fi le ') + assert waitforfiles(temp_dir + '/assets/ di r / fi le ') assert ( self.get(url='/%20di%20r%20/%20fi%20le%20')['body'] == 'blah' ), 'file name enclosing' try: - open(self.temp_dir + '/ф а', 'a').close() + open(temp_dir + '/ф а', 'a').close() utf8 = True except: @@ -138,38 +139,38 @@ class TestStatic(TestApplicationProto): if utf8: os.rename( - self.temp_dir + '/assets/ di r / fi le ', - self.temp_dir + '/assets/ di r /фа йл', + temp_dir + '/assets/ di r / fi le ', + temp_dir + '/assets/ di r /фа йл', ) - assert waitforfiles(self.temp_dir + '/assets/ di r /фа йл') + assert waitforfiles(temp_dir + '/assets/ di r /фа йл') assert ( self.get(url='/ di r /фа йл')['body'] == 'blah' ), 'file name 2' os.rename( - self.temp_dir + '/assets/ di r ', - self.temp_dir + '/assets/ди ректория', + temp_dir + '/assets/ di r ', + temp_dir + '/assets/ди ректория', ) - assert waitforfiles(self.temp_dir + '/assets/ди ректория/фа йл') + assert waitforfiles(temp_dir + '/assets/ди ректория/фа йл') assert ( self.get(url='/ди ректория/фа йл')['body'] == 'blah' ), 'dir name 2' - def test_static_unix_socket(self): + def test_static_unix_socket(self, temp_dir): sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.bind(self.temp_dir + '/assets/unix_socket') + sock.bind(temp_dir + '/assets/unix_socket') assert self.get(url='/unix_socket')['status'] == 404, 'socket' sock.close() - def test_static_unix_fifo(self): - os.mkfifo(self.temp_dir + '/assets/fifo') + def test_static_unix_fifo(self, temp_dir): + os.mkfifo(temp_dir + '/assets/fifo') assert self.get(url='/fifo')['status'] == 404, 'fifo' - def test_static_symlink(self): - os.symlink(self.temp_dir + '/assets/dir', self.temp_dir + '/assets/link') + def test_static_symlink(self, temp_dir): + os.symlink(temp_dir + '/assets/dir', temp_dir + '/assets/link') assert self.get(url='/dir')['status'] == 301, 'dir' assert self.get(url='/dir/file')['status'] == 200, 'file' @@ -312,7 +313,7 @@ class TestStatic(TestApplicationProto): ), 'mime_types same extensions case insensitive' @pytest.mark.skip('not yet') - def test_static_mime_types_invalid(self): + def test_static_mime_types_invalid(self, temp_dir): assert 'error' in self.http( b"""PUT /config/settings/http/static/mime_types/%0%00% HTTP/1.1\r Host: localhost\r @@ -323,5 +324,5 @@ Content-Length: 6\r raw_resp=True, raw=True, sock_type='unix', - addr=self.temp_dir + '/control.unit.sock', + addr=temp_dir + '/control.unit.sock', ), 'mime_types invalid' diff --git a/test/test_tls.py b/test/test_tls.py index 518a834c..7be426b1 100644 --- a/test/test_tls.py +++ b/test/test_tls.py @@ -5,6 +5,7 @@ import subprocess import pytest +from conftest import option from conftest import skip_alert from unit.applications.tls import TestApplicationTLS @@ -13,7 +14,7 @@ class TestTLS(TestApplicationTLS): prerequisites = {'modules': {'python': 'any', 'openssl': 'any'}} def findall(self, pattern): - with open(self.temp_dir + '/unit.log', 'r', errors='ignore') as f: + with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f: return re.findall(pattern, f.read()) def openssl_date_to_sec_epoch(self, date): @@ -134,7 +135,7 @@ class TestTLS(TestApplicationTLS): self.conf_get('/certificates/default/key') == 'RSA (2048 bits)' ), 'certificate key rsa' - def test_tls_certificate_key_ec(self): + def test_tls_certificate_key_ec(self, temp_dir): self.load('empty') self.openssl_conf() @@ -146,7 +147,7 @@ class TestTLS(TestApplicationTLS): '-noout', '-genkey', '-out', - self.temp_dir + '/ec.key', + temp_dir + '/ec.key', '-name', 'prime256v1', ], @@ -162,11 +163,11 @@ class TestTLS(TestApplicationTLS): '-subj', '/CN=ec/', '-config', - self.temp_dir + '/openssl.conf', + temp_dir + '/openssl.conf', '-key', - self.temp_dir + '/ec.key', + temp_dir + '/ec.key', '-out', - self.temp_dir + '/ec.crt', + temp_dir + '/ec.crt', ], stderr=subprocess.STDOUT, ) @@ -208,7 +209,7 @@ class TestTLS(TestApplicationTLS): == 2592000 ), 'certificate validity until' - def test_tls_certificate_chain(self): + def test_tls_certificate_chain(self, temp_dir): self.load('empty') self.certificate('root', False) @@ -221,11 +222,11 @@ class TestTLS(TestApplicationTLS): '-subj', '/CN=int/', '-config', - self.temp_dir + '/openssl.conf', + temp_dir + '/openssl.conf', '-out', - self.temp_dir + '/int.csr', + temp_dir + '/int.csr', '-keyout', - self.temp_dir + '/int.key', + temp_dir + '/int.key', ], stderr=subprocess.STDOUT, ) @@ -238,16 +239,16 @@ class TestTLS(TestApplicationTLS): '-subj', '/CN=end/', '-config', - self.temp_dir + '/openssl.conf', + temp_dir + '/openssl.conf', '-out', - self.temp_dir + '/end.csr', + temp_dir + '/end.csr', '-keyout', - self.temp_dir + '/end.key', + temp_dir + '/end.key', ], stderr=subprocess.STDOUT, ) - with open(self.temp_dir + '/ca.conf', 'w') as f: + with open(temp_dir + '/ca.conf', 'w') as f: f.write( """[ ca ] default_ca = myca @@ -267,16 +268,16 @@ commonName = supplied [ myca_extensions ] basicConstraints = critical,CA:TRUE""" % { - 'dir': self.temp_dir, - 'database': self.temp_dir + '/certindex', - 'certserial': self.temp_dir + '/certserial', + 'dir': temp_dir, + 'database': temp_dir + '/certindex', + 'certserial': temp_dir + '/certserial', } ) - with open(self.temp_dir + '/certserial', 'w') as f: + with open(temp_dir + '/certserial', 'w') as f: f.write('1000') - with open(self.temp_dir + '/certindex', 'w') as f: + with open(temp_dir + '/certindex', 'w') as f: f.write('') subprocess.call( @@ -287,15 +288,15 @@ basicConstraints = critical,CA:TRUE""" '-subj', '/CN=int/', '-config', - self.temp_dir + '/ca.conf', + temp_dir + '/ca.conf', '-keyfile', - self.temp_dir + '/root.key', + temp_dir + '/root.key', '-cert', - self.temp_dir + '/root.crt', + temp_dir + '/root.crt', '-in', - self.temp_dir + '/int.csr', + temp_dir + '/int.csr', '-out', - self.temp_dir + '/int.crt', + temp_dir + '/int.crt', ], stderr=subprocess.STDOUT, ) @@ -308,22 +309,22 @@ basicConstraints = critical,CA:TRUE""" '-subj', '/CN=end/', '-config', - self.temp_dir + '/ca.conf', + temp_dir + '/ca.conf', '-keyfile', - self.temp_dir + '/int.key', + temp_dir + '/int.key', '-cert', - self.temp_dir + '/int.crt', + temp_dir + '/int.crt', '-in', - self.temp_dir + '/end.csr', + temp_dir + '/end.csr', '-out', - self.temp_dir + '/end.crt', + temp_dir + '/end.crt', ], stderr=subprocess.STDOUT, ) - crt_path = self.temp_dir + '/end-int.crt' - end_path = self.temp_dir + '/end.crt' - int_path = self.temp_dir + '/int.crt' + crt_path = temp_dir + '/end-int.crt' + end_path = temp_dir + '/end.crt' + int_path = temp_dir + '/int.crt' with open(crt_path, 'wb') as crt, open(end_path, 'rb') as end, open( int_path, 'rb' @@ -333,7 +334,7 @@ basicConstraints = critical,CA:TRUE""" self.context = ssl.create_default_context() self.context.check_hostname = False self.context.verify_mode = ssl.CERT_REQUIRED - self.context.load_verify_locations(self.temp_dir + '/root.crt') + self.context.load_verify_locations(temp_dir + '/root.crt') # incomplete chain diff --git a/test/test_upstreams_rr.py b/test/test_upstreams_rr.py index 2ecf1d9a..c20d6054 100644 --- a/test/test_upstreams_rr.py +++ b/test/test_upstreams_rr.py @@ -9,8 +9,6 @@ class TestUpstreamsRR(TestApplicationPython): prerequisites = {'modules': {'python': 'any'}} def setup_method(self): - super().setup_method() - assert 'success' in self.conf( { "listeners": { @@ -391,9 +389,9 @@ Connection: close assert sum(resps) == 100, 'post sum' assert abs(resps[0] - resps[1]) <= self.cpu_count, 'post' - def test_upstreams_rr_unix(self): - addr_0 = self.temp_dir + '/sock_0' - addr_1 = self.temp_dir + '/sock_1' + def test_upstreams_rr_unix(self, temp_dir): + addr_0 = temp_dir + '/sock_0' + addr_1 = temp_dir + '/sock_1' assert 'success' in self.conf( { diff --git a/test/test_usr1.py b/test/test_usr1.py index 2e48c18f..3e44e4c5 100644 --- a/test/test_usr1.py +++ b/test/test_usr1.py @@ -1,6 +1,7 @@ import os from subprocess import call +from conftest import unit_stop from conftest import waitforfiles from unit.applications.lang.python import TestApplicationPython @@ -8,12 +9,12 @@ from unit.applications.lang.python import TestApplicationPython class TestUSR1(TestApplicationPython): prerequisites = {'modules': {'python': 'any'}} - def test_usr1_access_log(self): + def test_usr1_access_log(self, temp_dir): self.load('empty') log = 'access.log' log_new = 'new.log' - log_path = self.temp_dir + '/' + log + log_path = temp_dir + '/' + log assert 'success' in self.conf( '"' + log_path + '"', 'access_log' @@ -21,7 +22,7 @@ class TestUSR1(TestApplicationPython): assert waitforfiles(log_path), 'open' - os.rename(log_path, self.temp_dir + '/' + log_new) + os.rename(log_path, temp_dir + '/' + log_new) assert self.get()['status'] == 200 @@ -31,7 +32,7 @@ class TestUSR1(TestApplicationPython): ), 'rename new' assert not os.path.isfile(log_path), 'rename old' - with open(self.temp_dir + '/unit.pid', 'r') as f: + with open(temp_dir + '/unit.pid', 'r') as f: pid = f.read().rstrip() call(['kill', '-s', 'USR1', pid]) @@ -40,7 +41,7 @@ class TestUSR1(TestApplicationPython): assert self.get(url='/usr1')['status'] == 200 - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET /usr1 HTTP/1.1" 200 0 "-" "-"', log) @@ -48,12 +49,12 @@ class TestUSR1(TestApplicationPython): ), 'reopen 2' assert self.search_in_log(r'/usr1', log_new) is None, 'rename new 2' - def test_usr1_unit_log(self): + def test_usr1_unit_log(self, temp_dir): self.load('log_body') log_new = 'new.log' - log_path = self.temp_dir + '/unit.log' - log_path_new = self.temp_dir + '/' + log_new + log_path = temp_dir + '/unit.log' + log_path_new = temp_dir + '/' + log_new os.rename(log_path, log_path_new) @@ -63,7 +64,7 @@ class TestUSR1(TestApplicationPython): assert self.wait_for_record(body, log_new) is not None, 'rename new' assert not os.path.isfile(log_path), 'rename old' - with open(self.temp_dir + '/unit.pid', 'r') as f: + with open(temp_dir + '/unit.pid', 'r') as f: pid = f.read().rstrip() call(['kill', '-s', 'USR1', pid]) @@ -73,7 +74,7 @@ class TestUSR1(TestApplicationPython): body = 'body_for_a_log_unit' assert self.post(body=body)['status'] == 200 - self.stop() + unit_stop() assert self.wait_for_record(body) is not None, 'rename new' assert self.search_in_log(body, log_new) is None, 'rename new 2' diff --git a/test/test_variables.py b/test/test_variables.py index c458b636..bbb8f769 100644 --- a/test/test_variables.py +++ b/test/test_variables.py @@ -5,8 +5,6 @@ class TestVariables(TestApplicationProto): prerequisites = {} def setup_method(self): - super().setup_method() - assert 'success' in self.conf( { "listeners": {"*:7080": {"pass": "routes/$method"}}, diff --git a/test/unit/applications/lang/go.py b/test/unit/applications/lang/go.py index 7715bd6c..518cd017 100644 --- a/test/unit/applications/lang/go.py +++ b/test/unit/applications/lang/go.py @@ -7,8 +7,8 @@ from unit.applications.proto import TestApplicationProto class TestApplicationGo(TestApplicationProto): def prepare_env(self, script, name, static=False): - if not os.path.exists(self.temp_dir + '/go'): - os.mkdir(self.temp_dir + '/go') + if not os.path.exists(option.temp_dir + '/go'): + os.mkdir(option.temp_dir + '/go') env = os.environ.copy() env['GOPATH'] = option.current_dir + '/build/go' @@ -22,7 +22,7 @@ class TestApplicationGo(TestApplicationProto): '-ldflags', '-extldflags "-static"', '-o', - self.temp_dir + '/go/' + name, + option.temp_dir + '/go/' + name, option.test_dir + '/go/' + script + '/' + name + '.go', ] else: @@ -30,7 +30,7 @@ class TestApplicationGo(TestApplicationProto): 'go', 'build', '-o', - self.temp_dir + '/go/' + name, + option.temp_dir + '/go/' + name, option.test_dir + '/go/' + script + '/' + name + '.go', ] @@ -47,7 +47,7 @@ class TestApplicationGo(TestApplicationProto): static_build = False wdir = option.test_dir + "/go/" + script - executable = self.temp_dir + "/go/" + name + executable = option.temp_dir + "/go/" + name if 'isolation' in kwargs and 'rootfs' in kwargs['isolation']: wdir = "/go/" diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py index 01cbfa0b..a034d9a4 100644 --- a/test/unit/applications/lang/java.py +++ b/test/unit/applications/lang/java.py @@ -10,7 +10,7 @@ from unit.applications.proto import TestApplicationProto class TestApplicationJava(TestApplicationProto): def load(self, script, name='app', **kwargs): - app_path = self.temp_dir + '/java' + app_path = option.temp_dir + '/java' web_inf_path = app_path + '/WEB-INF/' classes_path = web_inf_path + 'classes/' script_path = option.test_dir + '/java/' + script + '/' diff --git a/test/unit/applications/lang/node.py b/test/unit/applications/lang/node.py index 877fc461..4aa9eb1c 100644 --- a/test/unit/applications/lang/node.py +++ b/test/unit/applications/lang/node.py @@ -11,17 +11,17 @@ class TestApplicationNode(TestApplicationProto): # copy application shutil.copytree( - option.test_dir + '/node/' + script, self.temp_dir + '/node' + option.test_dir + '/node/' + script, option.temp_dir + '/node' ) # copy modules shutil.copytree( option.current_dir + '/node/node_modules', - self.temp_dir + '/node/node_modules', + option.temp_dir + '/node/node_modules', ) - public_dir(self.temp_dir + '/node') + public_dir(option.temp_dir + '/node') self._load_conf( { @@ -32,7 +32,7 @@ class TestApplicationNode(TestApplicationProto): script: { "type": "external", "processes": {"spare": 0}, - "working_directory": self.temp_dir + '/node', + "working_directory": option.temp_dir + '/node', "executable": name, } }, diff --git a/test/unit/applications/lang/python.py b/test/unit/applications/lang/python.py index 47b95dac..a4a27bb9 100644 --- a/test/unit/applications/lang/python.py +++ b/test/unit/applications/lang/python.py @@ -12,7 +12,6 @@ class TestApplicationPython(TestApplicationProto): load_module = "wsgi" def load(self, script, name=None, module=None, **kwargs): - print() if name is None: name = script diff --git a/test/unit/applications/proto.py b/test/unit/applications/proto.py index 2f748c21..9a0361d5 100644 --- a/test/unit/applications/proto.py +++ b/test/unit/applications/proto.py @@ -14,7 +14,7 @@ class TestApplicationProto(TestControl): return time.mktime(time.strptime(date, template)) def search_in_log(self, pattern, name='unit.log'): - with open(self.temp_dir + '/' + name, 'r', errors='ignore') as f: + with open(option.temp_dir + '/' + name, 'r', errors='ignore') as f: return re.search(pattern, f.read()) def wait_for_record(self, pattern, name='unit.log'): diff --git a/test/unit/applications/tls.py b/test/unit/applications/tls.py index fdf681ae..fb1b112c 100644 --- a/test/unit/applications/tls.py +++ b/test/unit/applications/tls.py @@ -8,8 +8,6 @@ from unit.applications.proto import TestApplicationProto class TestApplicationTLS(TestApplicationProto): def setup_method(self): - super().setup_method() - self.context = ssl.create_default_context() self.context.check_hostname = False self.context.verify_mode = ssl.CERT_NONE @@ -24,9 +22,9 @@ class TestApplicationTLS(TestApplicationProto): '-x509', '-new', '-subj', '/CN=' + name + '/', - '-config', self.temp_dir + '/openssl.conf', - '-out', self.temp_dir + '/' + name + '.crt', - '-keyout', self.temp_dir + '/' + name + '.key', + '-config', option.temp_dir + '/openssl.conf', + '-out', option.temp_dir + '/' + name + '.crt', + '-keyout', option.temp_dir + '/' + name + '.key', ], stderr=subprocess.STDOUT, ) @@ -38,8 +36,8 @@ class TestApplicationTLS(TestApplicationProto): if key is None: key = crt - key_path = self.temp_dir + '/' + key + '.key' - crt_path = self.temp_dir + '/' + crt + '.crt' + key_path = option.temp_dir + '/' + key + '.key' + crt_path = option.temp_dir + '/' + crt + '.crt' with open(key_path, 'rb') as k, open(crt_path, 'rb') as c: return self.conf(k.read() + c.read(), '/certificates/' + crt) @@ -66,7 +64,7 @@ class TestApplicationTLS(TestApplicationProto): return ssl.get_server_certificate(addr, ssl_version=ssl_version) def openssl_conf(self): - conf_path = self.temp_dir + '/openssl.conf' + conf_path = option.temp_dir + '/openssl.conf' if os.path.exists(conf_path): return diff --git a/test/unit/control.py b/test/unit/control.py index 6fd350f4..f05aa827 100644 --- a/test/unit/control.py +++ b/test/unit/control.py @@ -1,5 +1,6 @@ import json +from conftest import option from unit.http import TestHTTP @@ -53,7 +54,7 @@ class TestControl(TestHTTP): args = { 'url': url, 'sock_type': 'unix', - 'addr': self.temp_dir + '/control.unit.sock', + 'addr': option.temp_dir + '/control.unit.sock', } if conf is not None: diff --git a/test/unit/feature/isolation.py b/test/unit/feature/isolation.py index c6f6f3c0..235177ab 100644 --- a/test/unit/feature/isolation.py +++ b/test/unit/feature/isolation.py @@ -21,6 +21,16 @@ class TestFeatureIsolation(TestApplicationProto): if 'go' in available['modules']: module = TestApplicationGo() + elif 'python' in available['modules']: + module = TestApplicationPython() + + elif 'php' in available['modules']: + module = TestApplicationPHP() + app = 'phpinfo' + + elif 'ruby' in available['modules']: + module = TestApplicationRuby() + elif 'java' in available['modules']: module = TestApplicationJava() @@ -32,16 +42,6 @@ class TestFeatureIsolation(TestApplicationProto): module = TestApplicationPerl() app = 'body_empty' - elif 'php' in available['modules']: - module = TestApplicationPHP() - app = 'phpinfo' - - elif 'python' in available['modules']: - module = TestApplicationPython() - - elif 'ruby' in available['modules']: - module = TestApplicationRuby() - if not module: return diff --git a/test/unit/http.py b/test/unit/http.py index 7845f9a8..5f073439 100644 --- a/test/unit/http.py +++ b/test/unit/http.py @@ -5,7 +5,6 @@ import os import re import select import socket -import time import pytest from conftest import option @@ -283,23 +282,6 @@ class TestHTTP(TestUnit): def getjson(self, **kwargs): return self.get(json=True, **kwargs) - def waitforsocket(self, port): - ret = False - - for i in range(50): - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect(('127.0.0.1', port)) - ret = True - break - except: - sock.close() - time.sleep(0.1) - - sock.close() - - assert ret, 'socket connected' - def form_encode(self, fields): is_multipart = False diff --git a/test/unit/main.py b/test/unit/main.py index d5940995..488b3f4d 100644 --- a/test/unit/main.py +++ b/test/unit/main.py @@ -1,55 +1,19 @@ -import atexit -import os -import re -import shutil -import signal -import stat -import subprocess -import tempfile -import time -from multiprocessing import Process - import pytest -from conftest import _check_alerts -from conftest import _print_log from conftest import option -from conftest import public_dir -from conftest import waitforfiles class TestUnit(): @classmethod def setup_class(cls, complete_check=True): - cls.available = option.available - unit = TestUnit() - - unit._run() - - # read unit.log - - for i in range(50): - with open(unit.temp_dir + '/unit.log', 'r') as f: - log = f.read() - m = re.search('controller started', log) - - if m is None: - time.sleep(0.1) - else: - break - - if m is None: - _print_log(path=unit.temp_dir + '/unit.log') - exit("Unit is writing log too long") - - def check(available, prerequisites): + def check(): missed = [] # check modules - if 'modules' in prerequisites: - available_modules = list(available['modules'].keys()) + if 'modules' in cls.prerequisites: + available_modules = list(option.available['modules'].keys()) - for module in prerequisites['modules']: + for module in cls.prerequisites['modules']: if module in available_modules: continue @@ -60,10 +24,10 @@ class TestUnit(): # check features - if 'features' in prerequisites: - available_features = list(available['features'].keys()) + if 'features' in cls.prerequisites: + available_features = list(option.available['features'].keys()) - for feature in prerequisites['features']: + for feature in cls.prerequisites['features']: if feature in available_features: continue @@ -72,132 +36,7 @@ class TestUnit(): if missed: pytest.skip(', '.join(missed) + ' feature(s) not supported') - def destroy(): - unit.stop() - _check_alerts(log) - shutil.rmtree(unit.temp_dir) - - def complete(): - destroy() - check(cls.available, cls.prerequisites) - if complete_check: - complete() - else: - unit.complete = complete - return unit - - def setup_method(self): - self._run() - - def _run(self): - build_dir = option.current_dir + '/build' - self.unitd = build_dir + '/unitd' - - if not os.path.isfile(self.unitd): - exit("Could not find unit") - - self.temp_dir = tempfile.mkdtemp(prefix='unit-test-') - - public_dir(self.temp_dir) - - if oct(stat.S_IMODE(os.stat(build_dir).st_mode)) != '0o777': - public_dir(build_dir) - - os.mkdir(self.temp_dir + '/state') - - with open(self.temp_dir + '/unit.log', 'w') as log: - self._p = subprocess.Popen( - [ - self.unitd, - '--no-daemon', - '--modules', build_dir, - '--state', self.temp_dir + '/state', - '--pid', self.temp_dir + '/unit.pid', - '--log', self.temp_dir + '/unit.log', - '--control', 'unix:' + self.temp_dir + '/control.unit.sock', - '--tmp', self.temp_dir, - ], - stderr=log, - ) - - atexit.register(self.stop) - - if not waitforfiles(self.temp_dir + '/control.unit.sock'): - _print_log(path=self.temp_dir + '/unit.log') - exit("Could not start unit") - - self._started = True - - def teardown_method(self): - self.stop() - - # check unit.log for alerts - - unit_log = self.temp_dir + '/unit.log' - - with open(unit_log, 'r', encoding='utf-8', errors='ignore') as f: - _check_alerts(f.read()) - - # remove unit.log - - if not option.save_log: - shutil.rmtree(self.temp_dir) + check() else: - _print_log(path=self.temp_dir) - - assert self.stop_errors == [None, None], 'stop errors' - - def stop(self): - if not self._started: - return - - self.stop_errors = [] - - self.stop_errors.append(self._stop()) - - self.stop_errors.append(self.stop_processes()) - - atexit.unregister(self.stop) - - self._started = False - - def _stop(self): - if self._p.poll() is not None: - return - - with self._p as p: - p.send_signal(signal.SIGQUIT) - - try: - retcode = p.wait(15) - if retcode: - return 'Child process terminated with code ' + str(retcode) - except: - p.kill() - return 'Could not terminate unit' - - def run_process(self, target, *args): - if not hasattr(self, '_processes'): - self._processes = [] - - process = Process(target=target, args=args) - process.start() - - self._processes.append(process) - - def stop_processes(self): - if not hasattr(self, '_processes'): - return - - fail = False - for process in self._processes: - if process.is_alive(): - process.terminate() - process.join(timeout=15) - - if process.is_alive(): - fail = True - - if fail: - return 'Fail to stop process' + return check -- cgit From 84136eb49ded615d7408d656ca9791542a0280a0 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Mon, 26 Oct 2020 22:24:32 +0300 Subject: Configure: using comma instead of space for passing -rpath value. This variant will be more interoperable across various systems and it's already used in Ruby module. Otherwise, configure tests fail on NetBSD with: gcc: Missing argument for -Wl,-rpath --- auto/modules/java | 2 +- auto/modules/php | 3 +-- auto/modules/python | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/auto/modules/java b/auto/modules/java index be8f443c..90b28b06 100644 --- a/auto/modules/java +++ b/auto/modules/java @@ -178,7 +178,7 @@ fi NXT_JAVA_LIB_SERVER_PATH="${NXT_JAVA_LIB_PATH}/server" -NXT_JAVA_LDFLAGS="-L${NXT_JAVA_LIB_SERVER_PATH} -Wl,-rpath ${NXT_JAVA_LIB_SERVER_PATH} -ljvm" +NXT_JAVA_LDFLAGS="-L${NXT_JAVA_LIB_SERVER_PATH} -Wl,-rpath,${NXT_JAVA_LIB_SERVER_PATH} -ljvm" nxt_found=no diff --git a/auto/modules/php b/auto/modules/php index 41eeb1c3..e92a67cd 100644 --- a/auto/modules/php +++ b/auto/modules/php @@ -111,8 +111,7 @@ if /bin/sh -c "${NXT_PHP_CONFIG} --version" >> $NXT_AUTOCONF_ERR 2>&1; then if [ "$NXT_PHP_LIB_PATH" != "" ]; then # "php-config --ldflags" does not contain path to libphp, but # contains usually path to libraries required by extensions. - NXT_PHP_LDFLAGS="-L${NXT_PHP_LIB_PATH} \ - -Wl,-rpath ${NXT_PHP_LIB_PATH}" + NXT_PHP_LDFLAGS="-L${NXT_PHP_LIB_PATH} -Wl,-rpath,${NXT_PHP_LIB_PATH}" fi fi diff --git a/auto/modules/python b/auto/modules/python index 48f6e5ef..2ab3ff7d 100644 --- a/auto/modules/python +++ b/auto/modules/python @@ -73,7 +73,7 @@ if /bin/sh -c "$NXT_PYTHON_CONFIG --prefix" >> $NXT_AUTOCONF_ERR 2>&1; then if [ "$NXT_PYTHON_LIB_PATH" != "" ]; then # "python-config --ldflags" may not contain path to libpython. - NXT_PYTHON_LDFLAGS="-L$NXT_PYTHON_LIB_PATH -Wl,-rpath $NXT_PYTHON_LIB_PATH" + NXT_PYTHON_LDFLAGS="-L$NXT_PYTHON_LIB_PATH -Wl,-rpath,$NXT_PYTHON_LIB_PATH" else NXT_PYTHON_LDFLAGS="" fi -- cgit From 434c3228d9bbdc31ff15be470a814344e5f13a4e Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Mon, 26 Oct 2020 22:26:02 +0300 Subject: Increased request memory pool size. Previous value was too small, which reduced efficiency of the pool causing a lot of additional allocations even for simple request and response. --- src/nxt_http_request.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index 76fb3427..650c1a89 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -220,7 +220,7 @@ nxt_http_request_create(nxt_task_t *task) nxt_buf_t *last; nxt_http_request_t *r; - mp = nxt_mp_create(1024, 128, 256, 32); + mp = nxt_mp_create(4096, 128, 512, 32); if (nxt_slow_path(mp == NULL)) { return NULL; } -- cgit From 8956e668cc3b4088a7bdd3d82ca99ac1445c8814 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Tue, 27 Oct 2020 04:09:52 +0000 Subject: Tests: fixed isolation detection. --- test/unit/applications/lang/java.py | 12 +++- test/unit/applications/lang/node.py | 5 +- test/unit/feature/isolation.py | 122 +++++++++++++++++++++++++++++------- 3 files changed, 112 insertions(+), 27 deletions(-) diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py index a034d9a4..0aa1a7af 100644 --- a/test/unit/applications/lang/java.py +++ b/test/unit/applications/lang/java.py @@ -9,7 +9,7 @@ from unit.applications.proto import TestApplicationProto class TestApplicationJava(TestApplicationProto): - def load(self, script, name='app', **kwargs): + def prepare_env(self, script): app_path = option.temp_dir + '/java' web_inf_path = app_path + '/WEB-INF/' classes_path = web_inf_path + 'classes/' @@ -75,6 +75,9 @@ class TestApplicationJava(TestApplicationProto): except: pytest.fail('Cann\'t run javac process.') + def load(self, script, **kwargs): + self.prepare_env(script) + self._load_conf( { "listeners": {"*:7080": {"pass": "applications/" + script}}, @@ -83,8 +86,11 @@ class TestApplicationJava(TestApplicationProto): "unit_jars": option.current_dir + '/build', "type": 'java', "processes": {"spare": 0}, - "working_directory": script_path, - "webapp": app_path, + "working_directory": option.test_dir + + '/java/' + + script + + '/', + "webapp": option.temp_dir + '/java', } }, }, diff --git a/test/unit/applications/lang/node.py b/test/unit/applications/lang/node.py index 4aa9eb1c..98fd9ffc 100644 --- a/test/unit/applications/lang/node.py +++ b/test/unit/applications/lang/node.py @@ -7,7 +7,7 @@ from unit.applications.proto import TestApplicationProto class TestApplicationNode(TestApplicationProto): - def load(self, script, name='app.js', **kwargs): + def prepare_env(self, script): # copy application shutil.copytree( @@ -23,6 +23,9 @@ class TestApplicationNode(TestApplicationProto): public_dir(option.temp_dir + '/node') + def load(self, script, name='app.js', **kwargs): + self.prepare_env(script) + self._load_conf( { "listeners": { diff --git a/test/unit/feature/isolation.py b/test/unit/feature/isolation.py index 235177ab..7877c03a 100644 --- a/test/unit/feature/isolation.py +++ b/test/unit/feature/isolation.py @@ -3,11 +3,8 @@ import os from unit.applications.lang.go import TestApplicationGo from unit.applications.lang.java import TestApplicationJava from unit.applications.lang.node import TestApplicationNode -from unit.applications.lang.perl import TestApplicationPerl -from unit.applications.lang.php import TestApplicationPHP -from unit.applications.lang.python import TestApplicationPython -from unit.applications.lang.ruby import TestApplicationRuby from unit.applications.proto import TestApplicationProto +from conftest import option class TestFeatureIsolation(TestApplicationProto): @@ -16,40 +13,119 @@ class TestFeatureIsolation(TestApplicationProto): def check(self, available, temp_dir): test_conf = {"namespaces": {"credential": True}} - module = '' - app = 'empty' + conf = '' if 'go' in available['modules']: - module = TestApplicationGo() + TestApplicationGo().prepare_env('empty', 'app') + + conf = { + "listeners": {"*:7080": {"pass": "applications/empty"}}, + "applications": { + "empty": { + "type": "external", + "processes": {"spare": 0}, + "working_directory": option.test_dir + "/go/empty", + "executable": option.temp_dir + "/go/app", + "isolation": {"namespaces": {"credential": True}}, + }, + }, + } elif 'python' in available['modules']: - module = TestApplicationPython() + conf = { + "listeners": {"*:7080": {"pass": "applications/empty"}}, + "applications": { + "empty": { + "type": "python", + "processes": {"spare": 0}, + "path": option.test_dir + "/python/empty", + "working_directory": option.test_dir + "/python/empty", + "module": "wsgi", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } elif 'php' in available['modules']: - module = TestApplicationPHP() - app = 'phpinfo' + conf = { + "listeners": {"*:7080": {"pass": "applications/phpinfo"}}, + "applications": { + "phpinfo": { + "type": "php", + "processes": {"spare": 0}, + "root": option.test_dir + "/php/phpinfo", + "working_directory": option.test_dir + "/php/phpinfo", + "index": "index.php", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } elif 'ruby' in available['modules']: - module = TestApplicationRuby() + conf = { + "listeners": {"*:7080": {"pass": "applications/empty"}}, + "applications": { + "empty": { + "type": "ruby", + "processes": {"spare": 0}, + "working_directory": option.test_dir + "/ruby/empty", + "script": option.test_dir + "/ruby/empty/config.ru", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } elif 'java' in available['modules']: - module = TestApplicationJava() + TestApplicationJava().prepare_env('empty') + + conf = { + "listeners": {"*:7080": {"pass": "applications/empty"}}, + "applications": { + "empty": { + "unit_jars": option.current_dir + "/build", + "type": "java", + "processes": {"spare": 0}, + "working_directory": option.test_dir + "/java/empty/", + "webapp": option.temp_dir + "/java", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } elif 'node' in available['modules']: - module = TestApplicationNode() - app = 'basic' + TestApplicationNode().prepare_env('basic') + + conf = { + "listeners": {"*:7080": {"pass": "applications/basic"}}, + "applications": { + "basic": { + "type": "external", + "processes": {"spare": 0}, + "working_directory": option.temp_dir + "/node", + "executable": "app.js", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } elif 'perl' in available['modules']: - module = TestApplicationPerl() - app = 'body_empty' - - if not module: + conf = { + "listeners": {"*:7080": {"pass": "applications/body_empty"}}, + "applications": { + "body_empty": { + "type": "perl", + "processes": {"spare": 0}, + "working_directory": option.test_dir + + "/perl/body_empty", + "script": option.test_dir + "/perl/body_empty/psgi.pl", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } + + else: return - module.temp_dir = temp_dir - module.load(app) - - resp = module.conf(test_conf, 'applications/' + app + '/isolation') - if 'success' not in resp: + if 'success' not in self.conf(conf): return userns = self.getns('user') -- cgit From 779b1131c56539ed74ab9db9a03f7bff71e203ba Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 28 Oct 2020 00:01:46 +0300 Subject: Router: closing app worker's ports. --- src/nxt_router.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/nxt_router.c b/src/nxt_router.c index a3218047..bbd9a54e 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -4667,6 +4667,25 @@ nxt_router_free_app(nxt_task_t *task, void *obj, void *data) nxt_port_use(task, port, -1); } + nxt_thread_mutex_lock(&app->mutex); + + for ( ;; ) { + port = nxt_port_hash_retrieve(&app->port_hash); + if (port == NULL) { + break; + } + + app->port_hash_count--; + + port->app = NULL; + + nxt_port_close(task, port); + + nxt_port_use(task, port, -1); + } + + nxt_thread_mutex_unlock(&app->mutex); + nxt_assert(app->processes == 0); nxt_assert(app->active_requests == 0); nxt_assert(app->port_hash_count == 0); -- cgit From 00561a961f8c7727556271e424e1e425ac9a88ce Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 28 Oct 2020 00:01:46 +0300 Subject: Libunit: protecting the new mmap from being used in another thread. Until the mmap is received by the router, only the creator thread may use this mmap, so the "mmap not found" state in the router is avoided. --- src/nxt_unit.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/nxt_unit.c b/src/nxt_unit.c index f75d61bc..65a76bee 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -314,6 +314,7 @@ struct nxt_unit_ctx_impl_s { struct nxt_unit_mmap_s { nxt_port_mmap_header_t *hdr; + pthread_t src_thread; /* of nxt_unit_read_buf_t */ nxt_queue_t awaiting_rbuf; @@ -3389,7 +3390,10 @@ retry: for (mm = lib->outgoing.elts; mm < mm_end; mm++) { hdr = mm->hdr; - if (hdr->sent_over != 0xFFFFu && hdr->sent_over != port->id.id) { + if (hdr->sent_over != 0xFFFFu + && (hdr->sent_over != port->id.id + || mm->src_thread != pthread_self())) + { continue; } @@ -3657,6 +3661,7 @@ nxt_unit_new_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, int n) hdr->src_pid = lib->pid; hdr->dst_pid = port->id.pid; hdr->sent_over = port->id.id; + mm->src_thread = pthread_self(); /* Mark first n chunk(s) as busy */ for (i = 0; i < n; i++) { -- cgit From 38a9027fe54edcfc8157174a8ce575ed7acefa79 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 28 Oct 2020 00:01:46 +0300 Subject: Router: checking a buffer before accessing its memory fields. This fixes the router's crash on buildbot; the reason was an unexpected 'last' response from the application to the router arriving before the response headers. The last buffer is not a memory buffer, so the result of accessing memory fields is unpredictable. The unexpected 'last' message was caused by an error in libunit; fixed in fee8fd855a00. --- src/nxt_router.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index bbd9a54e..6277bd47 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -3787,7 +3787,7 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_http_request_send_body(task, r, NULL); } else { - size_t b_size = nxt_buf_mem_used_size(&b->mem); + size_t b_size = nxt_buf_is_mem(b) ? nxt_buf_mem_used_size(&b->mem) : 0; if (nxt_slow_path(b_size < sizeof(*resp))) { goto fail; -- cgit From 735bb2f1276a7d768cffd1e780114a10018980cb Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 28 Oct 2020 00:01:46 +0300 Subject: Added error response logging. Every internal server error response should have a clear description in log. --- src/nxt_router.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 6277bd47..15706428 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -3722,6 +3722,7 @@ static void nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { + size_t b_size, count; nxt_int_t ret; nxt_app_t *app; nxt_buf_t *b, *next; @@ -3787,15 +3788,20 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_http_request_send_body(task, r, NULL); } else { - size_t b_size = nxt_buf_is_mem(b) ? nxt_buf_mem_used_size(&b->mem) : 0; + b_size = nxt_buf_is_mem(b) ? nxt_buf_mem_used_size(&b->mem) : 0; - if (nxt_slow_path(b_size < sizeof(*resp))) { + if (nxt_slow_path(b_size < sizeof(nxt_unit_response_t))) { + nxt_alert(task, "response buffer too small: %z", b_size); goto fail; } resp = (void *) b->mem.pos; - if (nxt_slow_path(b_size < sizeof(*resp) - + resp->fields_count * sizeof(nxt_unit_field_t))) { + count = (b_size - sizeof(nxt_unit_response_t)) + / sizeof(nxt_unit_field_t); + + if (nxt_slow_path(count < resp->fields_count)) { + nxt_alert(task, "response buffer too small for fields count: %D", + resp->fields_count); goto fail; } -- cgit From ccee391ab28d2742a98c612c42a37d6dcdbcd5f7 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 28 Oct 2020 00:01:46 +0300 Subject: Router: broadcasting the SHM_ACK message to all process ports. --- src/nxt_port_memory.c | 46 +++++++++++++++++++++++++++++++++++++--------- src/nxt_port_memory.h | 2 ++ src/nxt_router.c | 3 +-- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/nxt_port_memory.c b/src/nxt_port_memory.c index 1e01629e..ae9f079c 100644 --- a/src/nxt_port_memory.c +++ b/src/nxt_port_memory.c @@ -17,6 +17,10 @@ #include +static void nxt_port_broadcast_shm_ack(nxt_task_t *task, nxt_port_t *port, + void *data); + + nxt_inline void nxt_port_mmap_handler_use(nxt_port_mmap_handler_t *mmap_handler, int i) { @@ -112,7 +116,6 @@ nxt_port_mmap_buf_completion(nxt_task_t *task, void *obj, void *data) u_char *p; nxt_mp_t *mp; nxt_buf_t *b, *next; - nxt_port_t *port; nxt_process_t *process; nxt_chunk_id_t c; nxt_port_mmap_header_t *hdr; @@ -171,14 +174,7 @@ complete_buf: { process = nxt_runtime_process_find(task->thread->runtime, hdr->src_pid); - if (process != NULL && !nxt_queue_is_empty(&process->ports)) { - port = nxt_process_port_first(process); - - if (port->type == NXT_PROCESS_APP) { - (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_SHM_ACK, - -1, 0, 0, NULL); - } - } + nxt_process_broadcast_shm_ack(task, process); } release_buf: @@ -976,3 +972,35 @@ nxt_port_mmap_get_method(nxt_task_t *task, nxt_port_t *port, nxt_buf_t *b) return m; } + + +void +nxt_process_broadcast_shm_ack(nxt_task_t *task, nxt_process_t *process) +{ + nxt_port_t *port; + + if (nxt_slow_path(process == NULL || nxt_queue_is_empty(&process->ports))) + { + return; + } + + port = nxt_process_port_first(process); + + if (port->type == NXT_PROCESS_APP) { + nxt_port_post(task, port, nxt_port_broadcast_shm_ack, process); + } +} + + +static void +nxt_port_broadcast_shm_ack(nxt_task_t *task, nxt_port_t *port, void *data) +{ + nxt_process_t *process; + + process = data; + + nxt_queue_each(port, &process->ports, nxt_port_t, link) { + (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_SHM_ACK, + -1, 0, 0, NULL); + } nxt_queue_loop; +} diff --git a/src/nxt_port_memory.h b/src/nxt_port_memory.h index 8e71af3d..a2cdf5dd 100644 --- a/src/nxt_port_memory.h +++ b/src/nxt_port_memory.h @@ -71,4 +71,6 @@ nxt_port_mmap_get_method(nxt_task_t *task, nxt_port_t *port, nxt_buf_t *b); nxt_int_t nxt_shm_open(nxt_task_t *task, size_t size); +void nxt_process_broadcast_shm_ack(nxt_task_t *task, nxt_process_t *process); + #endif /* _NXT_PORT_MEMORY_H_INCLUDED_ */ diff --git a/src/nxt_router.c b/src/nxt_router.c index 15706428..cf627746 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -5389,8 +5389,7 @@ nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_thread_mutex_unlock(&process->incoming.mutex); if (ack) { - (void) nxt_port_socket_write(task, msg->port, NXT_PORT_MSG_SHM_ACK, - -1, 0, 0, NULL); + nxt_process_broadcast_shm_ack(task, process); } } -- cgit From 28ab1de364d048a4cb3f92179adebdd1eb851d65 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 28 Oct 2020 00:01:46 +0300 Subject: Libunit: gracefully quitting a multicontext application. --- src/nxt_unit.c | 96 +++++++++++++++++++++++++++++++++++++++++++--------------- src/nxt_unit.h | 2 ++ 2 files changed, 74 insertions(+), 24 deletions(-) diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 65a76bee..4b0f5230 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -305,6 +305,8 @@ struct nxt_unit_ctx_impl_s { /* of nxt_unit_read_buf_t */ nxt_queue_t free_rbuf; + int online; + nxt_unit_mmap_buf_t ctx_buf[2]; nxt_unit_read_buf_t ctx_read_buf; @@ -354,7 +356,6 @@ struct nxt_unit_impl_s { pid_t pid; int log_fd; - int online; nxt_unit_ctx_impl_t main_ctx; }; @@ -561,7 +562,6 @@ nxt_unit_create(nxt_unit_init_t *init) lib->ports.slot = NULL; lib->log_fd = STDERR_FILENO; - lib->online = 1; nxt_queue_init(&lib->contexts); @@ -615,10 +615,15 @@ nxt_unit_ctx_init(nxt_unit_impl_t *lib, nxt_unit_ctx_impl_t *ctx_impl, nxt_unit_lib_use(lib); + pthread_mutex_lock(&lib->mutex); + nxt_queue_insert_tail(&lib->contexts, &ctx_impl->link); + pthread_mutex_unlock(&lib->mutex); + ctx_impl->use_count = 1; ctx_impl->wait_items = 0; + ctx_impl->online = 1; nxt_queue_init(&ctx_impl->free_req); nxt_queue_init(&ctx_impl->free_ws); @@ -1119,7 +1124,7 @@ nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - if (new_port_msg->id == (nxt_port_id_t) -1) { + if (new_port_msg->id == NXT_UNIT_SHARED_PORT_ID) { nxt_unit_port_id_init(&new_port.id, lib->pid, new_port_msg->id); new_port.in_fd = recv_msg->fd[0]; @@ -1161,7 +1166,7 @@ nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) return NXT_UNIT_ERROR; } - if (new_port_msg->id == (nxt_port_id_t) -1) { + if (new_port_msg->id == NXT_UNIT_SHARED_PORT_ID) { lib->shared_port = port; } else { @@ -4408,18 +4413,20 @@ nxt_unit_process_pop_first(nxt_unit_impl_t *lib) int nxt_unit_run(nxt_unit_ctx_t *ctx) { - int rc; - nxt_unit_impl_t *lib; + int rc; + nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_ctx_use(ctx); - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + rc = NXT_UNIT_OK; - while (nxt_fast_path(lib->online)) { + while (nxt_fast_path(ctx_impl->online)) { rc = nxt_unit_run_once_impl(ctx); if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + nxt_unit_quit(ctx); break; } } @@ -4696,18 +4703,16 @@ int nxt_unit_run_ctx(nxt_unit_ctx_t *ctx) { int rc; - nxt_unit_impl_t *lib; nxt_unit_read_buf_t *rbuf; nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_ctx_use(ctx); - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); rc = NXT_UNIT_OK; - while (nxt_fast_path(lib->online)) { + while (nxt_fast_path(ctx_impl->online)) { rbuf = nxt_unit_read_buf_get(ctx); if (nxt_slow_path(rbuf == NULL)) { rc = NXT_UNIT_ERROR; @@ -4802,13 +4807,16 @@ nxt_unit_run_shared(nxt_unit_ctx_t *ctx) int rc; nxt_unit_impl_t *lib; nxt_unit_read_buf_t *rbuf; + nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_ctx_use(ctx); lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + rc = NXT_UNIT_OK; - while (nxt_fast_path(lib->online)) { + while (nxt_fast_path(ctx_impl->online)) { rbuf = nxt_unit_read_buf_get(ctx); if (nxt_slow_path(rbuf == NULL)) { rc = NXT_UNIT_ERROR; @@ -4867,6 +4875,7 @@ nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) int rc; nxt_unit_impl_t *lib; nxt_unit_read_buf_t *rbuf; + nxt_unit_ctx_impl_t *ctx_impl; rbuf = nxt_unit_read_buf_get(ctx); if (nxt_slow_path(rbuf == NULL)) { @@ -4874,6 +4883,7 @@ nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) } lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); retry: @@ -4906,7 +4916,7 @@ retry: return NXT_UNIT_ERROR; } - if (lib->online) { + if (ctx_impl->online) { goto retry; } @@ -4950,14 +4960,14 @@ nxt_unit_ctx_alloc(nxt_unit_ctx_t *ctx, void *data) queue_fd = -1; - port = nxt_unit_create_port(ctx); + port = nxt_unit_create_port(&new_ctx->ctx); if (nxt_slow_path(port == NULL)) { goto fail; } new_ctx->read_port = port; - queue_fd = nxt_unit_shm_open(ctx, sizeof(nxt_port_queue_t)); + queue_fd = nxt_unit_shm_open(&new_ctx->ctx, sizeof(nxt_port_queue_t)); if (nxt_slow_path(queue_fd == -1)) { goto fail; } @@ -4976,7 +4986,7 @@ nxt_unit_ctx_alloc(nxt_unit_ctx_t *ctx, void *data) port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); port_impl->queue = mem; - rc = nxt_unit_send_port(ctx, lib->router_port, port, queue_fd); + rc = nxt_unit_send_port(&new_ctx->ctx, lib->router_port, port, queue_fd); if (nxt_slow_path(rc != NXT_UNIT_OK)) { goto fail; } @@ -5041,8 +5051,12 @@ nxt_unit_ctx_free(nxt_unit_ctx_impl_t *ctx_impl) pthread_mutex_destroy(&ctx_impl->mutex); + pthread_mutex_lock(&lib->mutex); + nxt_queue_remove(&ctx_impl->link); + pthread_mutex_unlock(&lib->mutex); + if (nxt_fast_path(ctx_impl->read_port != NULL)) { nxt_unit_remove_port(lib, &ctx_impl->read_port->id); nxt_unit_port_release(ctx_impl->read_port); @@ -5229,7 +5243,7 @@ nxt_inline void nxt_unit_port_release(nxt_unit_port_t *port) } if (port_impl->queue != NULL) { - munmap(port_impl->queue, (port->id.id == (nxt_port_id_t) -1) + munmap(port_impl->queue, (port->id.id == NXT_UNIT_SHARED_PORT_ID) ? sizeof(nxt_app_queue_t) : sizeof(nxt_port_queue_t)); } @@ -5346,7 +5360,9 @@ nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *queue) goto unlock; } - if (port->id.id >= process->next_port_id) { + if (port->id.id != NXT_UNIT_SHARED_PORT_ID + && port->id.id >= process->next_port_id) + { process->next_port_id = port->id.id + 1; } @@ -5514,17 +5530,49 @@ nxt_unit_remove_process(nxt_unit_impl_t *lib, nxt_unit_process_t *process) static void nxt_unit_quit(nxt_unit_ctx_t *ctx) { - nxt_unit_impl_t *lib; + nxt_port_msg_t msg; + nxt_unit_impl_t *lib; + nxt_unit_ctx_impl_t *ctx_impl; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); - if (lib->online) { - lib->online = 0; + if (!ctx_impl->online) { + return; + } - if (lib->callbacks.quit != NULL) { - lib->callbacks.quit(ctx); - } + ctx_impl->online = 0; + + if (lib->callbacks.quit != NULL) { + lib->callbacks.quit(ctx); } + + if (ctx != &lib->main_ctx.ctx) { + return; + } + + memset(&msg, 0, sizeof(nxt_port_msg_t)); + + msg.pid = lib->pid; + msg.type = _NXT_PORT_MSG_QUIT; + + pthread_mutex_lock(&lib->mutex); + + nxt_queue_each(ctx_impl, &lib->contexts, nxt_unit_ctx_impl_t, link) { + + if (ctx == &ctx_impl->ctx + || ctx_impl->read_port == NULL + || ctx_impl->read_port->out_fd == -1) + { + continue; + } + + (void) nxt_unit_port_send(ctx, ctx_impl->read_port, + &msg, sizeof(msg), NULL, 0); + + } nxt_queue_loop; + + pthread_mutex_unlock(&lib->mutex); } diff --git a/src/nxt_unit.h b/src/nxt_unit.h index e90f0781..52f9bc27 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -34,6 +34,8 @@ enum { #define NXT_UNIT_INIT_ENV "NXT_UNIT_INIT" +#define NXT_UNIT_SHARED_PORT_ID ((uint16_t) 0xFFFFu) + /* * Mostly opaque structure with library state. * -- cgit From a5508cec7a55fe04ab66451c7510fab0e0d4577c Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 28 Oct 2020 00:01:46 +0300 Subject: Libunit: added a function to discern main and worker contexts. --- src/nxt_unit.c | 11 +++++++++++ src/nxt_unit.h | 2 ++ 2 files changed, 13 insertions(+) diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 4b0f5230..6a0657c6 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -4854,6 +4854,17 @@ nxt_unit_run_shared(nxt_unit_ctx_t *ctx) } +int +nxt_unit_is_main_ctx(nxt_unit_ctx_t *ctx) +{ + nxt_unit_impl_t *lib; + + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + + return (ctx == &lib->main_ctx.ctx); +} + + int nxt_unit_process_port_msg(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) { diff --git a/src/nxt_unit.h b/src/nxt_unit.h index 52f9bc27..303d5aa1 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -210,6 +210,8 @@ int nxt_unit_run_ctx(nxt_unit_ctx_t *ctx); int nxt_unit_run_shared(nxt_unit_ctx_t *ctx); +int nxt_unit_is_main_ctx(nxt_unit_ctx_t *ctx); + /* * Receive and process one message, invoke configured callbacks. * -- cgit From 131b6a7ffab7b3303a00a50f5cf764dc99e23cc0 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 28 Oct 2020 00:01:46 +0300 Subject: Libunit: releasing cached read buffers when destroying context. --- src/nxt_unit.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 6a0657c6..7d2bf2c7 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -5023,6 +5023,7 @@ nxt_unit_ctx_free(nxt_unit_ctx_impl_t *ctx_impl) { nxt_unit_impl_t *lib; nxt_unit_mmap_buf_t *mmap_buf; + nxt_unit_read_buf_t *rbuf; nxt_unit_request_info_impl_t *req_impl; nxt_unit_websocket_frame_impl_t *ws_impl; @@ -5060,6 +5061,13 @@ nxt_unit_ctx_free(nxt_unit_ctx_impl_t *ctx_impl) } nxt_queue_loop; + nxt_queue_each(rbuf, &ctx_impl->free_rbuf, nxt_unit_read_buf_t, link) + { + if (rbuf != &ctx_impl->ctx_read_buf) { + nxt_unit_free(&ctx_impl->ctx, rbuf); + } + } nxt_queue_loop; + pthread_mutex_destroy(&ctx_impl->mutex); pthread_mutex_lock(&lib->mutex); -- cgit From 4cb8aeb31a8cf47f6c61aaccb95bbbf47cbc2393 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 28 Oct 2020 00:01:46 +0300 Subject: Router: introducing the PORT_ACK message. The PORT_ACK message is the router's response to the application's NEW_PORT message. After receiving PORT_ACK, the application is safe to process requests using this port. This message avoids a racing condition when the application starts processing a request from the shared queue and sends REQ_HEADERS_ACK. The REQ_HEADERS_ACK message contains the application port ID as reply_port, which the router uses to send request data. When the application creates a new port, it immediately sends it to the main router thread. Because the request is processed outside the main thread, a racing condition can occur between the receipt of the new port in the main thread and the receipt of REQ_HEADERS_ACK in the worker router thread where the same port is specified as reply_port. --- src/nxt_port.h | 3 +++ src/nxt_router.c | 2 ++ src/nxt_unit.c | 37 ++++++++++++++++++++++++++++++++----- src/nxt_unit.h | 2 ++ 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/nxt_port.h b/src/nxt_port.h index 3ac8c735..5ece3bfa 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -26,6 +26,7 @@ struct nxt_port_handlers_s { nxt_port_handler_t change_file; nxt_port_handler_t new_port; nxt_port_handler_t get_port; + nxt_port_handler_t port_ack; nxt_port_handler_t mmap; nxt_port_handler_t get_mmap; @@ -84,6 +85,7 @@ typedef enum { _NXT_PORT_MSG_CHANGE_FILE = nxt_port_handler_idx(change_file), _NXT_PORT_MSG_NEW_PORT = nxt_port_handler_idx(new_port), _NXT_PORT_MSG_GET_PORT = nxt_port_handler_idx(get_port), + _NXT_PORT_MSG_PORT_ACK = nxt_port_handler_idx(port_ack), _NXT_PORT_MSG_MMAP = nxt_port_handler_idx(mmap), _NXT_PORT_MSG_GET_MMAP = nxt_port_handler_idx(get_mmap), @@ -120,6 +122,7 @@ typedef enum { NXT_PORT_MSG_CHANGE_FILE = nxt_msg_last(_NXT_PORT_MSG_CHANGE_FILE), NXT_PORT_MSG_NEW_PORT = nxt_msg_last(_NXT_PORT_MSG_NEW_PORT), NXT_PORT_MSG_GET_PORT = nxt_msg_last(_NXT_PORT_MSG_GET_PORT), + NXT_PORT_MSG_PORT_ACK = nxt_msg_last(_NXT_PORT_MSG_PORT_ACK), NXT_PORT_MSG_MMAP = nxt_msg_last(_NXT_PORT_MSG_MMAP) | NXT_PORT_MSG_SYNC, NXT_PORT_MSG_GET_MMAP = nxt_msg_last(_NXT_PORT_MSG_GET_MMAP), diff --git a/src/nxt_router.c b/src/nxt_router.c index cf627746..fbc9a4c8 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -688,6 +688,8 @@ nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) port->app = app; port->main_app_port = main_app_port; + + nxt_port_socket_write(task, port, NXT_PORT_MSG_PORT_ACK, -1, 0, 0, NULL); } diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 7d2bf2c7..d35a3307 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -57,6 +57,7 @@ static int nxt_unit_ready(nxt_unit_ctx_t *ctx, int ready_fd, uint32_t stream, static int nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf); static int nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg); +static int nxt_unit_ctx_ready(nxt_unit_ctx_t *ctx); static int nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg); static int nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, @@ -306,6 +307,7 @@ struct nxt_unit_ctx_impl_s { nxt_queue_t free_rbuf; int online; + int ready; nxt_unit_mmap_buf_t ctx_buf[2]; nxt_unit_read_buf_t ctx_read_buf; @@ -624,6 +626,7 @@ nxt_unit_ctx_init(nxt_unit_impl_t *lib, nxt_unit_ctx_impl_t *ctx_impl, ctx_impl->use_count = 1; ctx_impl->wait_items = 0; ctx_impl->online = 1; + ctx_impl->ready = 0; nxt_queue_init(&ctx_impl->free_req); nxt_queue_init(&ctx_impl->free_ws); @@ -996,6 +999,10 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) rc = nxt_unit_process_new_port(ctx, &recv_msg); break; + case _NXT_PORT_MSG_PORT_ACK: + rc = nxt_unit_ctx_ready(ctx); + break; + case _NXT_PORT_MSG_CHANGE_FILE: nxt_unit_debug(ctx, "#%"PRIu32": change_file: fd %d", port_msg->stream, recv_msg.fd[0]); @@ -1169,8 +1176,28 @@ nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) if (new_port_msg->id == NXT_UNIT_SHARED_PORT_ID) { lib->shared_port = port; - } else { - nxt_unit_port_release(port); + return nxt_unit_ctx_ready(ctx); + } + + nxt_unit_port_release(port); + + return NXT_UNIT_OK; +} + + +static int +nxt_unit_ctx_ready(nxt_unit_ctx_t *ctx) +{ + nxt_unit_impl_t *lib; + nxt_unit_ctx_impl_t *ctx_impl; + + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + ctx_impl->ready = 1; + + if (lib->callbacks.ready_handler) { + return lib->callbacks.ready_handler(ctx); } return NXT_UNIT_OK; @@ -4495,17 +4522,17 @@ nxt_unit_read_buf(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) nxt_unit_port_impl_t *port_impl; struct pollfd fds[2]; - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); - if (ctx_impl->wait_items > 0 || lib->shared_port == NULL) { - + if (ctx_impl->wait_items > 0 || ctx_impl->ready == 0) { return nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf); } port_impl = nxt_container_of(ctx_impl->read_port, nxt_unit_port_impl_t, port); + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + retry: if (port_impl->from_socket == 0) { diff --git a/src/nxt_unit.h b/src/nxt_unit.h index 303d5aa1..cb78e862 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -154,6 +154,8 @@ struct nxt_unit_callbacks_s { /* Receive data on port id. Optional. */ ssize_t (*port_recv)(nxt_unit_ctx_t *, nxt_unit_port_t *port, void *buf, size_t buf_size, void *oob, size_t oob_size); + + int (*ready_handler)(nxt_unit_ctx_t *); }; -- cgit From d8cc830ea0363009e40d6bf380db1147ad6fb41e Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 28 Oct 2020 00:01:46 +0300 Subject: Libunit: waking another context with the RPC_READY message. --- src/nxt_unit.c | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/nxt_unit.c b/src/nxt_unit.c index d35a3307..7e97c050 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -107,6 +107,8 @@ static int nxt_unit_get_outgoing_buf(nxt_unit_ctx_t *ctx, uint32_t min_size, nxt_unit_mmap_buf_t *mmap_buf, char *local_buf); static int nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd); +static void nxt_unit_awake_ctx(nxt_unit_ctx_t *ctx, + nxt_unit_ctx_impl_t *ctx_impl); static void nxt_unit_mmaps_init(nxt_unit_mmaps_t *mmaps); nxt_inline void nxt_unit_process_use(nxt_unit_process_t *process); nxt_inline void nxt_unit_process_release(nxt_unit_process_t *process); @@ -988,6 +990,10 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) switch (port_msg->type) { + case _NXT_PORT_MSG_RPC_READY: + rc = NXT_UNIT_OK; + break; + case _NXT_PORT_MSG_QUIT: nxt_unit_debug(ctx, "#%"PRIu32": quit", port_msg->stream); @@ -1068,7 +1074,7 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) break; default: - nxt_unit_debug(ctx, "#%"PRIu32": ignore message type: %d", + nxt_unit_alert(ctx, "#%"PRIu32": ignore message type: %d", port_msg->stream, (int) port_msg->type); rc = NXT_UNIT_ERROR; @@ -4012,12 +4018,40 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) nxt_atomic_fetch_add(&ctx_impl->wait_items, -1); + nxt_unit_awake_ctx(ctx, ctx_impl); + } nxt_queue_loop; return rc; } +static void +nxt_unit_awake_ctx(nxt_unit_ctx_t *ctx, nxt_unit_ctx_impl_t *ctx_impl) +{ + nxt_port_msg_t msg; + + if (nxt_fast_path(ctx == &ctx_impl->ctx)) { + return; + } + + if (nxt_slow_path(ctx_impl->read_port == NULL + || ctx_impl->read_port->out_fd == -1)) + { + nxt_unit_alert(ctx, "target context read_port is NULL or not writable"); + + return; + } + + memset(&msg, 0, sizeof(nxt_port_msg_t)); + + msg.type = _NXT_PORT_MSG_RPC_READY; + + (void) nxt_unit_port_send(ctx, ctx_impl->read_port, + &msg, sizeof(msg), NULL, 0); +} + + static void nxt_unit_mmaps_init(nxt_unit_mmaps_t *mmaps) { @@ -5390,6 +5424,8 @@ nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *queue) nxt_atomic_fetch_add(&ctx_impl->wait_items, -1); + nxt_unit_awake_ctx(ctx, ctx_impl); + } nxt_queue_loop; return old_port; -- cgit From 80a8cb835bb780cdb3047b232809c5dfd6e0e794 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 28 Oct 2020 00:01:46 +0300 Subject: Preserving the app port write socket. The socket is required for intercontextual communication in multithreaded apps. --- src/nxt_application.c | 2 +- src/nxt_external.c | 8 +++++++- src/nxt_process.c | 5 +++-- src/nxt_unit.c | 12 ++++++------ 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/nxt_application.c b/src/nxt_application.c index 6935346c..3d08643c 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -723,7 +723,7 @@ nxt_unit_default_init(nxt_task_t *task, nxt_unit_init_t *init) init->read_port.id.pid = my_port->pid; init->read_port.id.id = my_port->id; init->read_port.in_fd = my_port->pair[0]; - init->read_port.out_fd = -1; + init->read_port.out_fd = my_port->pair[1]; init->log_fd = 2; diff --git a/src/nxt_external.c b/src/nxt_external.c index 1adb839c..5703e294 100644 --- a/src/nxt_external.c +++ b/src/nxt_external.c @@ -101,18 +101,24 @@ nxt_external_start(nxt_task_t *task, nxt_process_data_t *data) return NXT_ERROR; } + rc = nxt_external_fd_no_cloexec(task, my_port->pair[1]); + if (nxt_slow_path(rc != NXT_OK)) { + return NXT_ERROR; + } + end = buf + sizeof(buf); p = nxt_sprintf(buf, end, "%s;%uD;" "%PI,%ud,%d;" "%PI,%ud,%d;" - "%PI,%ud,%d;" + "%PI,%ud,%d,%d;" "%d,%z,%Z", NXT_VERSION, my_port->process->stream, main_port->pid, main_port->id, main_port->pair[1], router_port->pid, router_port->id, router_port->pair[1], my_port->pid, my_port->id, my_port->pair[0], + my_port->pair[1], 2, conf->shm_limit); if (nxt_slow_path(p == end)) { diff --git a/src/nxt_process.c b/src/nxt_process.c index 9be7974f..87419313 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -248,8 +248,6 @@ nxt_process_setup(nxt_task_t *task, nxt_process_t *process) port = nxt_process_port_first(process); - nxt_port_write_close(port); - nxt_port_enable(task, port, init->port_handlers); ret = init->setup(task, process); @@ -272,6 +270,9 @@ nxt_process_setup(nxt_task_t *task, nxt_process_t *process) } ret = init->start(task, &process->data); + + nxt_port_write_close(port); + break; default: diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 7e97c050..23848f5b 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -780,7 +780,7 @@ nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *router_port, uint32_t *shm_limit) { int rc; - int ready_fd, router_fd, read_fd; + int ready_fd, router_fd, read_in_fd, read_out_fd; char *unit_init, *version_end; long version_length; int64_t ready_pid, router_pid, read_pid; @@ -812,15 +812,15 @@ nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *router_port, "%"PRIu32";" "%"PRId64",%"PRIu32",%d;" "%"PRId64",%"PRIu32",%d;" - "%"PRId64",%"PRIu32",%d;" + "%"PRId64",%"PRIu32",%d,%d;" "%d,%"PRIu32, &ready_stream, &ready_pid, &ready_id, &ready_fd, &router_pid, &router_id, &router_fd, - &read_pid, &read_id, &read_fd, + &read_pid, &read_id, &read_in_fd, &read_out_fd, log_fd, shm_limit); - if (nxt_slow_path(rc != 12)) { + if (nxt_slow_path(rc != 13)) { nxt_unit_alert(NULL, "failed to scan variables: %d", rc); return NXT_UNIT_ERROR; @@ -840,8 +840,8 @@ nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *router_port, nxt_unit_port_id_init(&read_port->id, (pid_t) read_pid, read_id); - read_port->in_fd = read_fd; - read_port->out_fd = -1; + read_port->in_fd = read_in_fd; + read_port->out_fd = read_out_fd; read_port->data = NULL; *stream = ready_stream; -- cgit From f007ad4dcfee0037cd86bf31804795e5f60cb2d9 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 28 Oct 2020 00:01:46 +0300 Subject: Added threading to the libunit test app. --- src/test/nxt_unit_app_test.c | 130 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 108 insertions(+), 22 deletions(-) diff --git a/src/test/nxt_unit_app_test.c b/src/test/nxt_unit_app_test.c index 4ecb8d6b..5b2521e8 100644 --- a/src/test/nxt_unit_app_test.c +++ b/src/test/nxt_unit_app_test.c @@ -3,9 +3,14 @@ * Copyright (C) NGINX, Inc. */ +#include +#include #include #include #include +#include +#include +#include #define CONTENT_TYPE "Content-Type" @@ -28,12 +33,106 @@ #define BODY " Body:\n" -static inline char * -copy(char *p, const void *src, uint32_t len) +static int ready_handler(nxt_unit_ctx_t *ctx); +static void *worker(void *main_ctx); +static void greeting_app_request_handler(nxt_unit_request_info_t *req); +static inline char *copy(char *p, const void *src, uint32_t len); + + +static int thread_count; +static pthread_t *threads; + + +int +main(int argc, char **argv) { - memcpy(p, src, len); + int i, err; + nxt_unit_ctx_t *ctx; + nxt_unit_init_t init; - return p + len; + if (argc == 3 && strcmp(argv[1], "-t") == 0) { + thread_count = atoi(argv[2]); + } + + memset(&init, 0, sizeof(nxt_unit_init_t)); + + init.callbacks.request_handler = greeting_app_request_handler; + init.callbacks.ready_handler = ready_handler; + + ctx = nxt_unit_init(&init); + if (ctx == NULL) { + return 1; + } + + err = nxt_unit_run(ctx); + + nxt_unit_debug(ctx, "main worker finished with %d code", err); + + if (thread_count > 1) { + for (i = 0; i < thread_count - 1; i++) { + err = pthread_join(threads[i], NULL); + + nxt_unit_debug(ctx, "join thread #%d: %d", i, err); + } + + nxt_unit_free(ctx, threads); + } + + nxt_unit_done(ctx); + + nxt_unit_debug(NULL, "main worker done"); + + return 0; +} + + +static int +ready_handler(nxt_unit_ctx_t *ctx) +{ + int i, err; + + nxt_unit_debug(ctx, "ready"); + + if (!nxt_unit_is_main_ctx(ctx) || thread_count <= 1) { + return NXT_UNIT_OK; + } + + threads = nxt_unit_malloc(ctx, sizeof(pthread_t) * (thread_count - 1)); + if (threads == NULL) { + return NXT_UNIT_ERROR; + } + + for (i = 0; i < thread_count - 1; i++) { + err = pthread_create(&threads[i], NULL, worker, ctx); + if (err != 0) { + return NXT_UNIT_ERROR; + } + } + + return NXT_UNIT_OK; +} + + +static void * +worker(void *main_ctx) +{ + int rc; + nxt_unit_ctx_t *ctx; + + ctx = nxt_unit_ctx_alloc(main_ctx, NULL); + if (ctx == NULL) { + return NULL; + } + + nxt_unit_debug(ctx, "start worker"); + + rc = nxt_unit_run(ctx); + + nxt_unit_debug(ctx, "worker finished with %d code", rc); + + nxt_unit_done(ctx); + + return NULL; } @@ -168,24 +267,11 @@ fail: nxt_unit_request_done(req, rc); } -int -main() -{ - nxt_unit_ctx_t *ctx; - nxt_unit_init_t init; - - memset(&init, 0, sizeof(nxt_unit_init_t)); - - init.callbacks.request_handler = greeting_app_request_handler; - ctx = nxt_unit_init(&init); - if (ctx == NULL) { - return 1; - } - - nxt_unit_run(ctx); - - nxt_unit_done(ctx); +static inline char * +copy(char *p, const void *src, uint32_t len) +{ + memcpy(p, src, len); - return 0; + return p + len; } -- cgit From 6a00bab41e2ebffe5f61f6fb9641162624db41d1 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 28 Oct 2020 00:01:46 +0300 Subject: Tests: improving get_application_type() and fixing its name. This patch also enables multiversion tests running for Java. --- test/conftest.py | 4 +++- test/unit/applications/lang/java.py | 4 +++- test/unit/applications/lang/perl.py | 6 +----- test/unit/applications/lang/php.py | 6 +----- test/unit/applications/lang/python.py | 7 +------ test/unit/applications/lang/ruby.py | 6 +----- test/unit/applications/proto.py | 9 ++++----- 7 files changed, 14 insertions(+), 28 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 0b9adf69..41d1bf00 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -75,7 +75,9 @@ def skip_alert(*alerts): def pytest_generate_tests(metafunc): cls = metafunc.cls - if not hasattr(cls, 'application_type'): + if (not hasattr(cls, 'application_type') + or cls.application_type == None + or cls.application_type == 'external'): return type = cls.application_type diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py index 0aa1a7af..fe0dde7a 100644 --- a/test/unit/applications/lang/java.py +++ b/test/unit/applications/lang/java.py @@ -9,6 +9,8 @@ from unit.applications.proto import TestApplicationProto class TestApplicationJava(TestApplicationProto): + application_type = "java" + def prepare_env(self, script): app_path = option.temp_dir + '/java' web_inf_path = app_path + '/WEB-INF/' @@ -84,7 +86,7 @@ class TestApplicationJava(TestApplicationProto): "applications": { script: { "unit_jars": option.current_dir + '/build', - "type": 'java', + "type": self.get_application_type(), "processes": {"spare": 0}, "working_directory": option.test_dir + '/java/' diff --git a/test/unit/applications/lang/perl.py b/test/unit/applications/lang/perl.py index a27c7649..9dc24ace 100644 --- a/test/unit/applications/lang/perl.py +++ b/test/unit/applications/lang/perl.py @@ -7,17 +7,13 @@ class TestApplicationPerl(TestApplicationProto): def load(self, script, name='psgi.pl', **kwargs): script_path = option.test_dir + '/perl/' + script - appication_type = self.get_appication_type() - - if appication_type is None: - appication_type = self.application_type self._load_conf( { "listeners": {"*:7080": {"pass": "applications/" + script}}, "applications": { script: { - "type": appication_type, + "type": self.get_application_type(), "processes": {"spare": 0}, "working_directory": script_path, "script": script_path + '/' + name, diff --git a/test/unit/applications/lang/php.py b/test/unit/applications/lang/php.py index 2d50df2e..619dfc93 100644 --- a/test/unit/applications/lang/php.py +++ b/test/unit/applications/lang/php.py @@ -7,17 +7,13 @@ class TestApplicationPHP(TestApplicationProto): def load(self, script, index='index.php', **kwargs): script_path = option.test_dir + '/php/' + script - appication_type = self.get_appication_type() - - if appication_type is None: - appication_type = self.application_type self._load_conf( { "listeners": {"*:7080": {"pass": "applications/" + script}}, "applications": { script: { - "type": appication_type, + "type": self.get_application_type(), "processes": {"spare": 0}, "root": script_path, "working_directory": script_path, diff --git a/test/unit/applications/lang/python.py b/test/unit/applications/lang/python.py index a4a27bb9..374b0f9c 100644 --- a/test/unit/applications/lang/python.py +++ b/test/unit/applications/lang/python.py @@ -34,11 +34,6 @@ class TestApplicationPython(TestApplicationProto): script_path = '/app/python/' + name - appication_type = self.get_appication_type() - - if appication_type is None: - appication_type = self.application_type - self._load_conf( { "listeners": { @@ -46,7 +41,7 @@ class TestApplicationPython(TestApplicationProto): }, "applications": { name: { - "type": appication_type, + "type": self.get_application_type(), "processes": {"spare": 0}, "path": script_path, "working_directory": script_path, diff --git a/test/unit/applications/lang/ruby.py b/test/unit/applications/lang/ruby.py index bc3cefc6..82d66e65 100644 --- a/test/unit/applications/lang/ruby.py +++ b/test/unit/applications/lang/ruby.py @@ -7,17 +7,13 @@ class TestApplicationRuby(TestApplicationProto): def load(self, script, name='config.ru', **kwargs): script_path = option.test_dir + '/ruby/' + script - appication_type = self.get_appication_type() - - if appication_type is None: - appication_type = self.application_type self._load_conf( { "listeners": {"*:7080": {"pass": "applications/" + script}}, "applications": { script: { - "type": appication_type, + "type": self.get_application_type(), "processes": {"spare": 0}, "working_directory": script_path, "script": script_path + '/' + name, diff --git a/test/unit/applications/proto.py b/test/unit/applications/proto.py index 9a0361d5..6e760c70 100644 --- a/test/unit/applications/proto.py +++ b/test/unit/applications/proto.py @@ -7,6 +7,8 @@ from unit.control import TestControl class TestApplicationProto(TestControl): + application_type = None + def sec_epoch(self): return time.mktime(time.gmtime()) @@ -28,15 +30,12 @@ class TestApplicationProto(TestControl): return found - def get_appication_type(self): + def get_application_type(self): current_test = ( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0] ) - if current_test in option.generated_tests: - return option.generated_tests[current_test] - - return None + return option.generated_tests.get(current_test, self.application_type) def _load_conf(self, conf, **kwargs): if 'applications' in conf: -- cgit From 5ffd88ad7c682f4bc60702d8829d1534aafb09e8 Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Thu, 29 Oct 2020 14:24:38 +0000 Subject: Isolation: correctly unmount non-dependent paths first. When mount points reside within other mount points, this patch sorts them by path length and then unmounts then in an order reverse to their mounting. This results in independent paths being unmounted first. This fixes an issue in buildbots where dependent paths failed to unmount, leading to the build script removing system-wide language libraries. --- src/nxt_isolation.c | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/nxt_isolation.c b/src/nxt_isolation.c index ac7a37e8..03160de3 100644 --- a/src/nxt_isolation.c +++ b/src/nxt_isolation.c @@ -41,6 +41,8 @@ static nxt_int_t nxt_isolation_set_mounts(nxt_task_t *task, nxt_process_t *process, nxt_str_t *app_type); static nxt_int_t nxt_isolation_set_lang_mounts(nxt_task_t *task, nxt_process_t *process, nxt_array_t *syspaths); +static int nxt_cdecl nxt_isolation_mount_compare(const void *v1, + const void *v2); static void nxt_isolation_unmount_all(nxt_task_t *task, nxt_process_t *process); #if (NXT_HAVE_PIVOT_ROOT) && (NXT_HAVE_CLONE_NEWNS) @@ -607,20 +609,48 @@ nxt_isolation_set_lang_mounts(nxt_task_t *task, nxt_process_t *process, } #endif + qsort(mounts->elts, mounts->nelts, sizeof(nxt_fs_mount_t), + nxt_isolation_mount_compare); + process->isolation.mounts = mounts; return NXT_OK; } +static int nxt_cdecl +nxt_isolation_mount_compare(const void *v1, const void *v2) +{ + const nxt_fs_mount_t *mnt1, *mnt2; + + mnt1 = v1; + mnt2 = v2; + + return nxt_strlen(mnt1->src) > nxt_strlen(mnt2->src); +} + + void nxt_isolation_unmount_all(nxt_task_t *task, nxt_process_t *process) { - size_t i, n; + size_t n; nxt_array_t *mounts; + nxt_runtime_t *rt; nxt_fs_mount_t *mnt; nxt_process_automount_t *automount; + rt = task->thread->runtime; + + if (!rt->capabilities.setid) { + return; + } + +#if (NXT_HAVE_CLONE_NEWNS) + if (nxt_is_clone_flag_set(process->isolation.clone.flags, NEWNS)) { + return; + } +#endif + nxt_debug(task, "unmount all (%s)", process->name); automount = &process->isolation.automount; @@ -628,12 +658,14 @@ nxt_isolation_unmount_all(nxt_task_t *task, nxt_process_t *process) n = mounts->nelts; mnt = mounts->elts; - for (i = 0; i < n; i++) { - if (mnt[i].builtin && !automount->language_deps) { + while (n > 0) { + n--; + + if (mnt[n].builtin && !automount->language_deps) { continue; } - nxt_fs_unmount(mnt[i].dst); + nxt_fs_unmount(mnt[n].dst); } } -- cgit From 417f5d911ddb3a46b590d89e73313856a32ff435 Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Thu, 29 Oct 2020 14:31:37 +0000 Subject: Tests: added new ruby isolation test without namespaces. --- test/test_ruby_isolation.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/test/test_ruby_isolation.py b/test/test_ruby_isolation.py index bf934540..79c94ba2 100644 --- a/test/test_ruby_isolation.py +++ b/test/test_ruby_isolation.py @@ -26,7 +26,7 @@ class TestRubyIsolation(TestApplicationRuby): return check if not complete_check else check() - def test_ruby_isolation_rootfs(self, is_su): + def test_ruby_isolation_rootfs_mount_namespace(self, is_su): isolation_features = option.available['features']['isolation'].keys() if 'mnt' not in isolation_features: @@ -55,3 +55,22 @@ class TestRubyIsolation(TestApplicationRuby): ) assert self.get()['status'] == 200, 'status int' + + def test_ruby_isolation_rootfs(self, is_su): + if not is_su: + pytest.skip('requires root') + return + + isolation = {'rootfs': option.test_dir} + + self.load('status_int', isolation=isolation) + + assert 'success' in self.conf( + '"/ruby/status_int/config.ru"', 'applications/status_int/script', + ) + + assert 'success' in self.conf( + '"/ruby/status_int"', 'applications/status_int/working_directory', + ) + + assert self.get()['status'] == 200, 'status int' -- cgit From 0390cb3a61051dd93e206d50591aff5759cf42fc Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Thu, 29 Oct 2020 20:30:53 +0000 Subject: Isolation: mounting of procfs by default when using "rootfs". --- auto/modules/java | 5 +- auto/modules/python | 2 +- auto/modules/ruby | 14 ++--- src/nxt_application.c | 17 +++--- src/nxt_fs.c | 113 ++++++++++++++++++++++++++++------- src/nxt_fs.h | 65 ++++++++------------ src/nxt_isolation.c | 86 ++++++++++++++------------ src/nxt_main_process.c | 10 +++- src/nxt_process.h | 2 +- test/test_go_isolation.py | 80 +++++++++++++++++-------- test/test_php_isolation.py | 62 +++++++++++-------- test/test_python_isolation.py | 52 +++++++++------- test/test_python_isolation_chroot.py | 2 +- test/test_ruby_isolation.py | 27 +++++---- test/unit/applications/lang/php.py | 14 +++++ 15 files changed, 353 insertions(+), 198 deletions(-) diff --git a/auto/modules/java b/auto/modules/java index 90b28b06..6996485c 100644 --- a/auto/modules/java +++ b/auto/modules/java @@ -326,11 +326,10 @@ cat << END > $NXT_BUILD_DIR/$NXT_JAVA_MOUNTS_HEADER static const nxt_fs_mount_t nxt_java_mounts[] = { - {(u_char *) "proc", (u_char *) "/proc", (u_char *) "proc", 0, NULL, 1}, {(u_char *) "$NXT_JAVA_LIBC_DIR", (u_char *) "$NXT_JAVA_LIBC_DIR", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, {(u_char *) "$NXT_JAVA_HOME", (u_char *) "$NXT_JAVA_HOME", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, }; diff --git a/auto/modules/python b/auto/modules/python index 2ab3ff7d..9be6b370 100644 --- a/auto/modules/python +++ b/auto/modules/python @@ -138,7 +138,7 @@ pyver = "python" + str(sys.version_info[0]) + "." + str(sys.version_info[1]) print("static const nxt_fs_mount_t nxt_python_mounts[] = {") -pattern = "{(u_char *) \"%s\", (u_char *) \"%s\", (u_char *) \"bind\", NXT_MS_BIND|NXT_MS_REC, NULL, 1}," +pattern = "{(u_char *) \"%s\", (u_char *) \"%s\", NXT_FS_BIND, (u_char *) \"bind\", 0, NULL, 1, 1}," base = None for p in sys.path: if len(p) > 0: diff --git a/auto/modules/ruby b/auto/modules/ruby index e0d54516..68324b44 100644 --- a/auto/modules/ruby +++ b/auto/modules/ruby @@ -156,23 +156,23 @@ cat << END > $NXT_RUBY_MOUNTS_PATH static const nxt_fs_mount_t nxt_ruby_mounts[] = { {(u_char *) "$NXT_RUBY_RUBYHDRDIR", (u_char *) "$NXT_RUBY_RUBYHDRDIR", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, {(u_char *) "$NXT_RUBY_ARCHHDRDIR", (u_char *) "$NXT_RUBY_ARCHHDRDIR", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, {(u_char *) "$NXT_RUBY_SITEDIR", (u_char *) "$NXT_RUBY_SITEDIR", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, {(u_char *) "$NXT_RUBY_LIBDIR", (u_char *) "$NXT_RUBY_LIBDIR", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, {(u_char *) "$NXT_RUBY_TOPDIR", (u_char *) "$NXT_RUBY_TOPDIR", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, {(u_char *) "$NXT_RUBY_PREFIXDIR", (u_char *) "$NXT_RUBY_PREFIXDIR", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, END for path in `echo $NXT_RUBY_GEMPATH | tr ':' '\n'`; do $echo "{(u_char *) \"$path\", (u_char *) \"$path\"," >> $NXT_RUBY_MOUNTS_PATH - $echo "(u_char *) \"bind\", NXT_MS_BIND | NXT_MS_REC, NULL, 1}," >> $NXT_RUBY_MOUNTS_PATH + $echo "NXT_FS_BIND, (u_char *) \"bind\", 0, NULL, 1, 1}," >> $NXT_RUBY_MOUNTS_PATH done $echo "};" >> $NXT_RUBY_MOUNTS_PATH diff --git a/src/nxt_application.c b/src/nxt_application.c index 3d08643c..5d58e60c 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -208,14 +208,14 @@ nxt_discovery_modules(nxt_task_t *task, const char *path) mounts = module[i].mounts; size += mounts->nelts * nxt_length("{\"src\": \"\", \"dst\": \"\", " - "\"fstype\": \"\", \"flags\": , " - "\"data\": \"\"},"); + "\"type\": , \"name\": \"\", " + "\"flags\": , \"data\": \"\"},"); mnt = mounts->elts; for (j = 0; j < mounts->nelts; j++) { size += nxt_strlen(mnt[j].src) + nxt_strlen(mnt[j].dst) - + nxt_strlen(mnt[j].fstype) + NXT_INT_T_LEN + + nxt_strlen(mnt[j].name) + (2 * NXT_INT_T_LEN) + (mnt[j].data == NULL ? 0 : nxt_strlen(mnt[j].data)); } } @@ -242,9 +242,10 @@ nxt_discovery_modules(nxt_task_t *task, const char *path) for (j = 0; j < mounts->nelts; j++) { p = nxt_sprintf(p, end, "{\"src\": \"%s\", \"dst\": \"%s\", " - "\"fstype\": \"%s\", \"flags\": %d, " + "\"name\": \"%s\", \"type\": %d, \"flags\": %d, " "\"data\": \"%s\"},", - mnt[j].src, mnt[j].dst, mnt[j].fstype, mnt[j].flags, + mnt[j].src, mnt[j].dst, mnt[j].name, mnt[j].type, + mnt[j].flags, mnt[j].data == NULL ? (u_char *) "" : mnt[j].data); } @@ -386,11 +387,13 @@ nxt_discovery_module(nxt_task_t *task, nxt_mp_t *mp, nxt_array_t *modules, goto fail; } - to->fstype = nxt_cstr_dup(mp, to->fstype, from->fstype); - if (nxt_slow_path(to->fstype == NULL)) { + to->name = nxt_cstr_dup(mp, to->name, from->name); + if (nxt_slow_path(to->name == NULL)) { goto fail; } + to->type = from->type; + if (from->data != NULL) { to->data = nxt_cstr_dup(mp, to->data, from->data); if (nxt_slow_path(to->data == NULL)) { diff --git a/src/nxt_fs.c b/src/nxt_fs.c index 0228c25a..87d3b4a5 100644 --- a/src/nxt_fs.c +++ b/src/nxt_fs.c @@ -18,15 +18,59 @@ static nxt_int_t nxt_fs_mkdir(const u_char *dir, mode_t mode); nxt_int_t nxt_fs_mount(nxt_task_t *task, nxt_fs_mount_t *mnt) { - int rc; + int rc; + const char *fsname; + unsigned long flags; - rc = mount((const char *) mnt->src, (const char *) mnt->dst, - (const char *) mnt->fstype, mnt->flags, mnt->data); + flags = 0; + + switch (mnt->type) { + case NXT_FS_BIND: + if (nxt_slow_path(mnt->flags != 0)) { + nxt_log(task, NXT_LOG_WARN, + "bind mount ignores additional flags"); + } + + fsname = "bind"; + flags = MS_BIND | MS_REC; + break; + + case NXT_FS_PROC: + fsname = "proc"; + goto getflags; + + case NXT_FS_TMP: + fsname = "tmpfs"; + goto getflags; + + default: + fsname = (const char *) mnt->name; + + getflags: + + if (mnt->flags & NXT_FS_FLAGS_NODEV) { + flags |= MS_NODEV; + } + + if (mnt->flags & NXT_FS_FLAGS_NOEXEC) { + flags |= MS_NOEXEC; + } + + if (mnt->flags & NXT_FS_FLAGS_NOSUID) { + flags |= MS_NOSUID; + } + + if (!(mnt->flags & NXT_FS_FLAGS_NOTIME)) { + flags |= MS_RELATIME; + } + } + + rc = mount((const char *) mnt->src, (const char *) mnt->dst, fsname, flags, + mnt->data); if (nxt_slow_path(rc < 0)) { - nxt_alert(task, "mount(\"%s\", \"%s\", \"%s\", %d, \"%s\") %E", - mnt->src, mnt->dst, mnt->fstype, mnt->flags, mnt->data, - nxt_errno); + nxt_alert(task, "mount(\"%s\", \"%s\", \"%s\", %ul, \"%s\") %E", + mnt->src, mnt->dst, fsname, flags, mnt->data, nxt_errno); return NXT_ERROR; } @@ -34,37 +78,66 @@ nxt_fs_mount(nxt_task_t *task, nxt_fs_mount_t *mnt) return NXT_OK; } - #elif (NXT_HAVE_FREEBSD_NMOUNT) nxt_int_t nxt_fs_mount(nxt_task_t *task, nxt_fs_mount_t *mnt) { + int flags; u_char *data, *p, *end; size_t iovlen; nxt_int_t ret; - const char *fstype; + const char *fsname; struct iovec iov[128]; char errmsg[256]; - if (nxt_strncmp(mnt->fstype, "bind", 4) == 0) { - fstype = "nullfs"; + if (nxt_slow_path((mnt->flags & NXT_FS_FLAGS_NODEV) && !mnt->builtin)) { + nxt_alert(task, "nmount(2) doesn't support \"nodev\" option"); - } else if (nxt_strncmp(mnt->fstype, "proc", 4) == 0) { - fstype = "procfs"; + return NXT_ERROR; + } - } else if (nxt_strncmp(mnt->fstype, "tmpfs", 5) == 0) { - fstype = "tmpfs"; + flags = 0; - } else { - nxt_alert(task, "mount type \"%s\" not implemented.", mnt->fstype); - return NXT_ERROR; + switch (mnt->type) { + case NXT_FS_BIND: + fsname = "nullfs"; + break; + + case NXT_FS_PROC: + fsname = "procfs"; + goto getflags; + + case NXT_FS_TMP: + fsname = "tmpfs"; + goto getflags; + + default: + fsname = (const char *) mnt->name; + + getflags: + + if (mnt->flags & NXT_FS_FLAGS_NOEXEC) { + flags |= MNT_NOEXEC; + } + + if (mnt->flags & NXT_FS_FLAGS_NOSUID) { + flags |= MNT_NOSUID; + } + + if (mnt->flags & NXT_FS_FLAGS_NOTIME) { + flags |= MNT_NOATIME; + } + + if (mnt->flags & NXT_FS_FLAGS_RDONLY) { + flags |= MNT_RDONLY; + } } iov[0].iov_base = (void *) "fstype"; iov[0].iov_len = 7; - iov[1].iov_base = (void *) fstype; - iov[1].iov_len = nxt_strlen(fstype) + 1; + iov[1].iov_base = (void *) fsname; + iov[1].iov_len = nxt_strlen(fsname) + 1; iov[2].iov_base = (void *) "fspath"; iov[2].iov_len = 7; iov[3].iov_base = (void *) mnt->dst; @@ -117,7 +190,7 @@ nxt_fs_mount(nxt_task_t *task, nxt_fs_mount_t *mnt) ret = NXT_OK; - if (nxt_slow_path(nmount(iov, iovlen, 0) < 0)) { + if (nxt_slow_path(nmount(iov, iovlen, flags) < 0)) { nxt_alert(task, "nmount(%p, %d, 0) %s", iov, iovlen, errmsg); ret = NXT_ERROR; } diff --git a/src/nxt_fs.h b/src/nxt_fs.h index bbd7ab9f..ff589979 100644 --- a/src/nxt_fs.h +++ b/src/nxt_fs.h @@ -6,50 +6,33 @@ #define _NXT_FS_H_INCLUDED_ -#ifdef MS_BIND -#define NXT_MS_BIND MS_BIND -#else -#define NXT_MS_BIND 0 -#endif - -#ifdef MS_REC -#define NXT_MS_REC MS_BIND -#else -#define NXT_MS_REC 0 -#endif - -#ifdef MS_NOSUID -#define NXT_MS_NOSUID MS_NOSUID -#else -#define NXT_MS_NOSUID 0 -#endif - -#ifdef MS_NOEXEC -#define NXT_MS_NOEXEC MS_NOEXEC -#else -#define NXT_MS_NOEXEC 0 -#endif - -#ifdef MS_RELATIME -#define NXT_MS_RELATIME MS_RELATIME -#else -#define NXT_MS_RELATIME 0 -#endif - -#ifdef MS_NODEV -#define NXT_MS_NODEV MS_NODEV -#else -#define NXT_MS_NODEV 0 -#endif +typedef enum { + NXT_FS_UNKNOWN = 0, + NXT_FS_BIND, + NXT_FS_TMP, + NXT_FS_PROC, + NXT_FS_LAST, +} nxt_fs_type_t; + + +typedef enum { + NXT_FS_FLAGS_NOSUID = 1 << 0, + NXT_FS_FLAGS_NOEXEC = 1 << 1, + NXT_FS_FLAGS_NOTIME = 1 << 2, + NXT_FS_FLAGS_NODEV = 1 << 3, + NXT_FS_FLAGS_RDONLY = 1 << 4, +} nxt_fs_flags_t; typedef struct { - u_char *src; - u_char *dst; - u_char *fstype; - nxt_int_t flags; - u_char *data; - nxt_uint_t builtin; /* 1-bit */ + u_char *src; + u_char *dst; + nxt_fs_type_t type; + u_char *name; + nxt_fs_flags_t flags; + u_char *data; + nxt_uint_t builtin; /* 1-bit */ + nxt_uint_t deps; /* 1-bit */ } nxt_fs_mount_t; diff --git a/src/nxt_isolation.c b/src/nxt_isolation.c index 03160de3..e0f169aa 100644 --- a/src/nxt_isolation.c +++ b/src/nxt_isolation.c @@ -87,15 +87,6 @@ nxt_isolation_main_prefork(nxt_task_t *task, nxt_process_t *process, } #endif -#if (NXT_HAVE_ISOLATION_ROOTFS) - if (process->isolation.rootfs != NULL) { - ret = nxt_isolation_set_mounts(task, process, &app_conf->type); - if (nxt_slow_path(ret != NXT_OK)) { - return ret; - } - } -#endif - if (cap_setid) { ret = nxt_process_creds_set(task, process, &app_conf->user, &app_conf->group); @@ -126,6 +117,29 @@ nxt_isolation_main_prefork(nxt_task_t *task, nxt_process_t *process, } } +#if (NXT_HAVE_ISOLATION_ROOTFS) + if (process->isolation.rootfs != NULL) { + nxt_int_t has_mnt; + + ret = nxt_isolation_set_mounts(task, process, &app_conf->type); + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + + has_mnt = 0; + +#if (NXT_HAVE_CLONE_NEWNS) + has_mnt = nxt_is_clone_flag_set(process->isolation.clone.flags, NEWNS); +#endif + + if (process->user_cred->uid == 0 && !has_mnt) { + nxt_log(task, NXT_LOG_WARN, + "setting user \"root\" with \"rootfs\" is unsafe without " + "\"mount\" namespace isolation"); + } + } +#endif + #if (NXT_HAVE_CLONE_NEWUSER) ret = nxt_isolation_vldt_creds(task, process); if (nxt_slow_path(ret != NXT_OK)) { @@ -568,10 +582,13 @@ nxt_isolation_set_lang_mounts(nxt_task_t *task, nxt_process_t *process, } mnt->src = (u_char *) "tmpfs"; - mnt->fstype = (u_char *) "tmpfs"; - mnt->flags = NXT_MS_NOSUID | NXT_MS_NODEV | NXT_MS_NOEXEC | NXT_MS_RELATIME; + mnt->name = (u_char *) "tmpfs"; + mnt->type = NXT_FS_TMP; + mnt->flags = (NXT_FS_FLAGS_NOSUID | NXT_FS_FLAGS_NODEV + | NXT_FS_FLAGS_NOEXEC); mnt->data = (u_char *) "size=1m,mode=777"; mnt->builtin = 1; + mnt->deps = 0; mnt->dst = nxt_mp_nget(mp, rootfs_len + nxt_length("/tmp") + 1); if (nxt_slow_path(mnt->dst == NULL)) { @@ -582,32 +599,27 @@ nxt_isolation_set_lang_mounts(nxt_task_t *task, nxt_process_t *process, p = nxt_cpymem(p, "/tmp", 4); *p = '\0'; -#if (NXT_HAVE_CLONE_NEWPID) && (NXT_HAVE_CLONE_NEWNS) - - if (nxt_is_clone_flag_set(process->isolation.clone.flags, NEWPID) - && nxt_is_clone_flag_set(process->isolation.clone.flags, NEWNS)) - { - mnt = nxt_array_add(mounts); - if (nxt_slow_path(mnt == NULL)) { - return NXT_ERROR; - } - - mnt->fstype = (u_char *) "proc"; - mnt->src = (u_char *) "proc"; + mnt = nxt_array_add(mounts); + if (nxt_slow_path(mnt == NULL)) { + return NXT_ERROR; + } - mnt->dst = nxt_mp_nget(mp, rootfs_len + nxt_length("/proc") + 1); - if (nxt_slow_path(mnt->dst == NULL)) { - return NXT_ERROR; - } + mnt->name = (u_char *) "proc"; + mnt->type = NXT_FS_PROC; + mnt->src = (u_char *) "none"; + mnt->dst = nxt_mp_nget(mp, rootfs_len + nxt_length("/proc") + 1); + if (nxt_slow_path(mnt->dst == NULL)) { + return NXT_ERROR; + } - p = nxt_cpymem(mnt->dst, rootfs, rootfs_len); - p = nxt_cpymem(p, "/proc", 5); - *p = '\0'; + p = nxt_cpymem(mnt->dst, rootfs, rootfs_len); + p = nxt_cpymem(p, "/proc", 5); + *p = '\0'; - mnt->data = (u_char *) ""; - mnt->flags = 0; - } -#endif + mnt->data = (u_char *) ""; + mnt->flags = NXT_FS_FLAGS_NOEXEC | NXT_FS_FLAGS_NOSUID; + mnt->builtin = 1; + mnt->deps = 0; qsort(mounts->elts, mounts->nelts, sizeof(nxt_fs_mount_t), nxt_isolation_mount_compare); @@ -661,7 +673,7 @@ nxt_isolation_unmount_all(nxt_task_t *task, nxt_process_t *process) while (n > 0) { n--; - if (mnt[n].builtin && !automount->language_deps) { + if (mnt[n].deps && !automount->language_deps) { continue; } @@ -690,11 +702,11 @@ nxt_isolation_prepare_rootfs(nxt_task_t *task, nxt_process_t *process) for (i = 0; i < n; i++) { dst = mnt[i].dst; - if (mnt[i].builtin && !automount->language_deps) { + if (mnt[i].deps && !automount->language_deps) { continue; } - if (nxt_slow_path(nxt_memcmp(mnt[i].fstype, "bind", 4) == 0 + if (nxt_slow_path(mnt[i].type == NXT_FS_BIND && stat((const char *) mnt[i].src, &st) != 0)) { nxt_log(task, NXT_LOG_WARN, "host path not found: %s", mnt[i].src); diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index d2edab1d..e4ec8b57 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -1163,9 +1163,14 @@ static nxt_conf_map_t nxt_app_lang_mounts_map[] = { offsetof(nxt_fs_mount_t, dst), }, { - nxt_string("fstype"), + nxt_string("name"), NXT_CONF_MAP_CSTRZ, - offsetof(nxt_fs_mount_t, fstype), + offsetof(nxt_fs_mount_t, name), + }, + { + nxt_string("type"), + NXT_CONF_MAP_INT, + offsetof(nxt_fs_mount_t, type), }, { nxt_string("flags"), @@ -1297,6 +1302,7 @@ nxt_main_port_modules_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } mnt->builtin = 1; + mnt->deps = 1; ret = nxt_conf_map_object(rt->mem_pool, value, nxt_app_lang_mounts_map, diff --git a/src/nxt_process.h b/src/nxt_process.h index d9b4dff1..ddadb08f 100644 --- a/src/nxt_process.h +++ b/src/nxt_process.h @@ -74,7 +74,7 @@ typedef struct { typedef struct { - uint8_t language_deps; /* 1-byte */ + uint8_t language_deps; /* 1-bit */ } nxt_process_automount_t; diff --git a/test/test_go_isolation.py b/test/test_go_isolation.py index 9a84aa25..c68925b9 100644 --- a/test/test_go_isolation.py +++ b/test/test_go_isolation.py @@ -226,13 +226,23 @@ class TestGoIsolation(TestApplicationGo): if not self.isolation_key('pid'): pytest.skip('pid namespace is not supported') - if not (is_su or self.isolation_key('unprivileged_userns_clone')): - pytest.skip('requires root or unprivileged_userns_clone') + if not is_su: + if not self.isolation_key('unprivileged_userns_clone'): + pytest.skip('unprivileged clone is not available') - self.load( - 'ns_inspect', - isolation={'namespaces': {'pid': True, 'credential': True}}, - ) + if not self.isolation_key('user'): + pytest.skip('user namespace is not supported') + + if not self.isolation_key('mnt'): + pytest.skip('mnt namespace is not supported') + + isolation = {'namespaces': {'pid': True}} + + if not is_su: + isolation['namespaces']['mount'] = True + isolation['namespaces']['credential'] = True + + self.load('ns_inspect', isolation=isolation) obj = self.getjson()['body'] @@ -269,17 +279,28 @@ class TestGoIsolation(TestApplicationGo): == option.available['features']['isolation'][ns] ), ('%s match' % ns) - def test_go_isolation_rootfs_container(self, temp_dir): - if not self.isolation_key('unprivileged_userns_clone'): - pytest.skip('unprivileged clone is not available') + def test_go_isolation_rootfs_container(self, is_su, temp_dir): + if not is_su: + if not self.isolation_key('unprivileged_userns_clone'): + pytest.skip('unprivileged clone is not available') - if not self.isolation_key('mnt'): - pytest.skip('mnt namespace is not supported') + if not self.isolation_key('user'): + pytest.skip('user namespace is not supported') - isolation = { - 'namespaces': {'mount': True, 'credential': True}, - 'rootfs': temp_dir, - } + if not self.isolation_key('mnt'): + pytest.skip('mnt namespace is not supported') + + if not self.isolation_key('pid'): + pytest.skip('pid namespace is not supported') + + isolation = {'rootfs': temp_dir} + + if not is_su: + isolation['namespaces'] = { + 'mount': True, + 'credential': True, + 'pid': True + } self.load('ns_inspect', isolation=isolation) @@ -311,17 +332,28 @@ class TestGoIsolation(TestApplicationGo): obj = self.getjson(url='/?file=/bin/sh')['body'] assert obj['FileExists'] == False, 'file should not exists' - def test_go_isolation_rootfs_default_tmpfs(self, temp_dir): - if not self.isolation_key('unprivileged_userns_clone'): - pytest.skip('unprivileged clone is not available') + def test_go_isolation_rootfs_default_tmpfs(self, is_su, temp_dir): + if not is_su: + if not self.isolation_key('unprivileged_userns_clone'): + pytest.skip('unprivileged clone is not available') - if not self.isolation_key('mnt'): - pytest.skip('mnt namespace is not supported') + if not self.isolation_key('user'): + pytest.skip('user namespace is not supported') - isolation = { - 'namespaces': {'mount': True, 'credential': True}, - 'rootfs': temp_dir, - } + if not self.isolation_key('mnt'): + pytest.skip('mnt namespace is not supported') + + if not self.isolation_key('pid'): + pytest.skip('pid namespace is not supported') + + isolation = {'rootfs': temp_dir} + + if not is_su: + isolation['namespaces'] = { + 'mount': True, + 'credential': True, + 'pid': True + } self.load('ns_inspect', isolation=isolation) diff --git a/test/test_php_isolation.py b/test/test_php_isolation.py index ac6942c6..cc660e04 100644 --- a/test/test_php_isolation.py +++ b/test/test_php_isolation.py @@ -26,57 +26,71 @@ class TestPHPIsolation(TestApplicationPHP): return check if not complete_check else check() - def test_php_isolation_rootfs(self, is_su): + def test_php_isolation_rootfs(self, is_su, temp_dir): isolation_features = option.available['features']['isolation'].keys() - if 'mnt' not in isolation_features: - pytest.skip('requires mnt ns') - if not is_su: - if 'user' not in isolation_features: - pytest.skip('requires unprivileged userns or root') - if not 'unprivileged_userns_clone' in isolation_features: pytest.skip('requires unprivileged userns or root') - isolation = { - 'namespaces': {'credential': not is_su, 'mount': True}, - 'rootfs': option.test_dir, - } + if 'user' not in isolation_features: + pytest.skip('user namespace is not supported') + + if 'mnt' not in isolation_features: + pytest.skip('mnt namespace is not supported') + + if 'pid' not in isolation_features: + pytest.skip('pid namespace is not supported') + + isolation = {'rootfs': temp_dir} + + if not is_su: + isolation['namespaces'] = { + 'mount': True, + 'credential': True, + 'pid': True + } self.load('phpinfo', isolation=isolation) assert 'success' in self.conf( - '"/php/phpinfo"', 'applications/phpinfo/root' + '"/app/php/phpinfo"', 'applications/phpinfo/root' ) assert 'success' in self.conf( - '"/php/phpinfo"', 'applications/phpinfo/working_directory' + '"/app/php/phpinfo"', 'applications/phpinfo/working_directory' ) assert self.get()['status'] == 200, 'empty rootfs' - def test_php_isolation_rootfs_extensions(self, is_su): + def test_php_isolation_rootfs_extensions(self, is_su, temp_dir): isolation_features = option.available['features']['isolation'].keys() if not is_su: - if 'user' not in isolation_features: - pytest.skip('requires unprivileged userns or root') - if not 'unprivileged_userns_clone' in isolation_features: pytest.skip('requires unprivileged userns or root') + if 'user' not in isolation_features: + pytest.skip('user namespace is not supported') + if 'mnt' not in isolation_features: - pytest.skip('requires mnt ns') + pytest.skip('mnt namespace is not supported') + + if 'pid' not in isolation_features: + pytest.skip('pid namespace is not supported') - isolation = { - 'rootfs': option.test_dir, - 'namespaces': {'credential': not is_su, 'mount': not is_su}, - } + isolation = {'rootfs': temp_dir} + + if not is_su: + isolation['namespaces'] = { + 'mount': True, + 'credential': True, + 'pid': True + } self.load('list-extensions', isolation=isolation) assert 'success' in self.conf( - '"/php/list-extensions"', 'applications/list-extensions/root' + '"/app/php/list-extensions"', 'applications/list-extensions/root' ) assert 'success' in self.conf( @@ -85,7 +99,7 @@ class TestPHPIsolation(TestApplicationPHP): ) assert 'success' in self.conf( - '"/php/list-extensions"', + '"/app/php/list-extensions"', 'applications/list-extensions/working_directory', ) diff --git a/test/test_python_isolation.py b/test/test_python_isolation.py index 34abd1df..1a157528 100644 --- a/test/test_python_isolation.py +++ b/test/test_python_isolation.py @@ -29,24 +29,27 @@ class TestPythonIsolation(TestApplicationPython): def test_python_isolation_rootfs(self, is_su, temp_dir): isolation_features = option.available['features']['isolation'].keys() - if 'mnt' not in isolation_features: - pytest.skip('requires mnt ns') - if not is_su: - if 'user' not in isolation_features: - pytest.skip('requires unprivileged userns or root') - if not 'unprivileged_userns_clone' in isolation_features: pytest.skip('requires unprivileged userns or root') - isolation = { - 'namespaces': {'credential': not is_su, 'mount': True}, - 'rootfs': temp_dir, - } + if 'user' not in isolation_features: + pytest.skip('user namespace is not supported') - self.load('empty', isolation=isolation) + if 'mnt' not in isolation_features: + pytest.skip('mnt namespace is not supported') - assert self.get()['status'] == 200, 'python rootfs' + if 'pid' not in isolation_features: + pytest.skip('pid namespace is not supported') + + isolation = {'rootfs': temp_dir} + + if not is_su: + isolation['namespaces'] = { + 'mount': True, + 'credential': True, + 'pid': True + } self.load('ns_inspect', isolation=isolation) @@ -57,7 +60,7 @@ class TestPythonIsolation(TestApplicationPython): assert ( self.getjson(url='/?path=/proc/self')['body']['FileExists'] - == False + == True ), 'no /proc/self' assert ( @@ -78,22 +81,31 @@ class TestPythonIsolation(TestApplicationPython): def test_python_isolation_rootfs_no_language_deps(self, is_su, temp_dir): isolation_features = option.available['features']['isolation'].keys() - if 'mnt' not in isolation_features: - pytest.skip('requires mnt ns') - if not is_su: - if 'user' not in isolation_features: - pytest.skip('requires unprivileged userns or root') - if not 'unprivileged_userns_clone' in isolation_features: pytest.skip('requires unprivileged userns or root') + if 'user' not in isolation_features: + pytest.skip('user namespace is not supported') + + if 'mnt' not in isolation_features: + pytest.skip('mnt namespace is not supported') + + if 'pid' not in isolation_features: + pytest.skip('pid namespace is not supported') + isolation = { - 'namespaces': {'credential': not is_su, 'mount': True}, 'rootfs': temp_dir, 'automount': {'language_deps': False} } + if not is_su: + isolation['namespaces'] = { + 'mount': True, + 'credential': True, + 'pid': True + } + self.load('empty', isolation=isolation) assert (self.get()['status'] != 200), 'disabled language_deps' diff --git a/test/test_python_isolation_chroot.py b/test/test_python_isolation_chroot.py index 6d178b2e..134d2b8a 100644 --- a/test/test_python_isolation_chroot.py +++ b/test/test_python_isolation_chroot.py @@ -28,7 +28,7 @@ class TestPythonIsolation(TestApplicationPython): assert ( self.getjson(url='/?path=/proc/self')['body']['FileExists'] - == False + == True ), 'no /proc/self' assert ( diff --git a/test/test_ruby_isolation.py b/test/test_ruby_isolation.py index 79c94ba2..69e25de9 100644 --- a/test/test_ruby_isolation.py +++ b/test/test_ruby_isolation.py @@ -29,20 +29,27 @@ class TestRubyIsolation(TestApplicationRuby): def test_ruby_isolation_rootfs_mount_namespace(self, is_su): isolation_features = option.available['features']['isolation'].keys() - if 'mnt' not in isolation_features: - pytest.skip('requires mnt ns') - if not is_su: - if 'user' not in isolation_features: - pytest.skip('requires unprivileged userns or root') - if not 'unprivileged_userns_clone' in isolation_features: pytest.skip('requires unprivileged userns or root') - isolation = { - 'namespaces': {'credential': not is_su, 'mount': True}, - 'rootfs': option.test_dir, - } + if 'user' not in isolation_features: + pytest.skip('user namespace is not supported') + + if 'mnt' not in isolation_features: + pytest.skip('mnt namespace is not supported') + + if 'pid' not in isolation_features: + pytest.skip('pid namespace is not supported') + + isolation = {'rootfs': option.test_dir} + + if not is_su: + isolation['namespaces'] = { + 'mount': True, + 'credential': True, + 'pid': True + } self.load('status_int', isolation=isolation) diff --git a/test/unit/applications/lang/php.py b/test/unit/applications/lang/php.py index 619dfc93..3dbb32f5 100644 --- a/test/unit/applications/lang/php.py +++ b/test/unit/applications/lang/php.py @@ -1,4 +1,7 @@ from conftest import option +import os +import shutil + from unit.applications.proto import TestApplicationProto @@ -8,6 +11,17 @@ class TestApplicationPHP(TestApplicationProto): def load(self, script, index='index.php', **kwargs): script_path = option.test_dir + '/php/' + script + if kwargs.get('isolation') and kwargs['isolation'].get('rootfs'): + rootfs = kwargs['isolation']['rootfs'] + + if not os.path.exists(rootfs + '/app/php/'): + os.makedirs(rootfs + '/app/php/') + + if not os.path.exists(rootfs + '/app/php/' + script): + shutil.copytree(script_path, rootfs + '/app/php/' + script) + + script_path = '/app/php/' + script + self._load_conf( { "listeners": {"*:7080": {"pass": "applications/" + script}}, -- cgit From 50af47fd7c03aecb5bb9f248cac8ccbdb65c9b4c Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Fri, 30 Oct 2020 07:55:26 +0300 Subject: Isolation: fixed passing custom options to nmount(). The "iov" array was filled incorrectly when custom mounting options were set. --- src/nxt_fs.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/nxt_fs.c b/src/nxt_fs.c index 87d3b4a5..71498f99 100644 --- a/src/nxt_fs.c +++ b/src/nxt_fs.c @@ -172,8 +172,10 @@ nxt_fs_mount(nxt_task_t *task, nxt_fs_mount_t *mnt) *end = '\0'; - iov[iovlen++].iov_base = (void *) p; - iov[iovlen++].iov_len = (end - p) + 1; + iov[iovlen].iov_base = (void *) p; + iov[iovlen].iov_len = (end - p) + 1; + + iovlen++; p = end + 1; @@ -182,8 +184,10 @@ nxt_fs_mount(nxt_task_t *task, nxt_fs_mount_t *mnt) *end = '\0'; } - iov[iovlen++].iov_base = (void *) p; - iov[iovlen++].iov_len = nxt_strlen(p) + 1; + iov[iovlen].iov_base = (void *) p; + iov[iovlen].iov_len = nxt_strlen(p) + 1; + + iovlen++; } while (end != NULL && nxt_nitems(iov) > (iovlen + 2)); } -- cgit From bbe4b97ca159bbff19e3c507635e0d81a68c695e Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Fri, 30 Oct 2020 17:33:36 +0300 Subject: Java: supporting jsp-file attribute for servlet. This closes #487 issue on GitHub. --- src/java/nginx/unit/Context.java | 66 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/java/nginx/unit/Context.java b/src/java/nginx/unit/Context.java index 5f7ec22f..5aa3a982 100644 --- a/src/java/nginx/unit/Context.java +++ b/src/java/nginx/unit/Context.java @@ -1214,6 +1214,16 @@ public class Context implements ServletContext, InitParams processXmlInitParam(reg, (Element) child_node); continue; } + + if (tag_name.equals("filter-name") + || tag_name.equals("#text") + || tag_name.equals("#comment")) + { + continue; + } + + log("processWebXml: tag '" + tag_name + "' for filter '" + + filter_name + "' is ignored"); } filters_.add(reg); @@ -1306,6 +1316,22 @@ public class Context implements ServletContext, InitParams reg.setLoadOnStartup(Integer.parseInt(child_node.getTextContent().trim())); continue; } + + if (tag_name.equals("jsp-file")) { + reg.setJspFile(child_node.getTextContent().trim()); + continue; + } + + if (tag_name.equals("servlet-name") + || tag_name.equals("display-name") + || tag_name.equals("#text") + || tag_name.equals("#comment")) + { + continue; + } + + log("processWebXml: tag '" + tag_name + "' for servlet '" + + servlet_name + "' is ignored"); } servlets_.add(reg); @@ -1888,6 +1914,7 @@ public class Context implements ServletContext, InitParams private boolean initialized_ = false; private final List filters_ = new ArrayList<>(); private boolean system_jsp_servlet_ = false; + private String jsp_file_; private MultipartConfigElement multipart_config_; public ServletReg(String name, Class servlet_class) @@ -1921,6 +1948,21 @@ public class Context implements ServletContext, InitParams trace("ServletReg.init(): " + getName()); + if (jsp_file_ != null) { + setInitParameter("jspFile", jsp_file_); + jsp_file_ = null; + + ServletReg jsp_servlet = name2servlet_.get("jsp"); + + if (jsp_servlet.servlet_class_ != null) { + servlet_class_ = jsp_servlet.servlet_class_; + } else { + setClassName(jsp_servlet.getClassName()); + } + + system_jsp_servlet_ = jsp_servlet.system_jsp_servlet_; + } + if (system_jsp_servlet_) { JasperInitializer ji = new JasperInitializer(); @@ -1972,6 +2014,10 @@ public class Context implements ServletContext, InitParams throw new IllegalStateException("Class already initialized"); } + if (jsp_file_ != null) { + throw new IllegalStateException("jsp-file already initialized"); + } + super.setClassName(class_name); } @@ -1985,11 +2031,31 @@ public class Context implements ServletContext, InitParams throw new IllegalStateException("Class already initialized"); } + if (jsp_file_ != null) { + throw new IllegalStateException("jsp-file already initialized"); + } + super.setClassName(servlet_class.getName()); servlet_class_ = servlet_class; getAnnotationMultipartConfig(); } + public void setJspFile(String jsp_file) throws IllegalStateException + { + if (servlet_ != null + || servlet_class_ != null + || getClassName() != null) + { + throw new IllegalStateException("Class already initialized"); + } + + if (jsp_file_ != null) { + throw new IllegalStateException("jsp-file already initialized"); + } + + jsp_file_ = jsp_file; + } + private void getAnnotationMultipartConfig() { if (servlet_class_ == null) { return; -- cgit From d03b217f33db21d9af28d58d92ba02c2a2e48f5e Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Sun, 1 Nov 2020 13:22:11 +0300 Subject: Fixed building test app without debug. Compilers complained about unused variables after 37e2a3ea1bf1. --- src/test/nxt_unit_app_test.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/test/nxt_unit_app_test.c b/src/test/nxt_unit_app_test.c index 5b2521e8..b6dd13d2 100644 --- a/src/test/nxt_unit_app_test.c +++ b/src/test/nxt_unit_app_test.c @@ -72,7 +72,13 @@ main(int argc, char **argv) for (i = 0; i < thread_count - 1; i++) { err = pthread_join(threads[i], NULL); - nxt_unit_debug(ctx, "join thread #%d: %d", i, err); + if (nxt_fast_path(err == 0)) { + nxt_unit_debug(ctx, "join thread #%d", i); + + } else { + nxt_unit_alert(ctx, "pthread_join(#%d) failed: %s (%d)", + i, strerror(err), err); + } } nxt_unit_free(ctx, threads); @@ -132,7 +138,7 @@ worker(void *main_ctx) nxt_unit_done(ctx); - return NULL; + return (void *) (intptr_t) rc; } -- cgit From 4ba9e1d0058bc1710a0078af6e9a1f7e4bee986a Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Mon, 2 Nov 2020 21:05:04 +0300 Subject: Java: upgrading 3rd-party components. --- auto/modules/java | 8 ++++---- auto/modules/java_jar.sha512 | 28 ++++++++++++++-------------- test/unit/applications/lang/java.py | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/auto/modules/java b/auto/modules/java index 6996485c..a1f27131 100644 --- a/auto/modules/java +++ b/auto/modules/java @@ -238,7 +238,7 @@ cat << END > $NXT_JAVA_JARS static const char *nxt_java_system_jars[] = { END -NXT_TOMCAT_VERSION=9.0.13 +NXT_TOMCAT_VERSION=9.0.39 NXT_JAR_VERSION=$NXT_TOMCAT_VERSION @@ -271,7 +271,7 @@ NXT_JAR_NAME=tomcat-util . auto/modules/java_get_jar NXT_JAR_NAME=ecj -NXT_JAR_VERSION=3.13.102 +NXT_JAR_VERSION=3.23.0 NXT_JAR_NAMESPACE=org/eclipse/jdt/ . auto/modules/java_get_jar @@ -284,7 +284,7 @@ static const char *nxt_java_unit_jars[] = { "$NXT_UNIT_JAR", END -NXT_JAR_VERSION=9.4.12.v20180830 +NXT_JAR_VERSION=9.4.33.v20201020 NXT_JAR_NAMESPACE=org/eclipse/jetty/ NXT_JAR_NAME=jetty-util @@ -297,7 +297,7 @@ NXT_JAR_NAME=jetty-http . auto/modules/java_get_jar NXT_JAR_NAME=classgraph -NXT_JAR_VERSION=4.4.11 +NXT_JAR_VERSION=4.8.90 NXT_JAR_NAMESPACE=io/github/classgraph/ . auto/modules/java_get_jar diff --git a/auto/modules/java_jar.sha512 b/auto/modules/java_jar.sha512 index 0f6daa8e..214cff33 100644 --- a/auto/modules/java_jar.sha512 +++ b/auto/modules/java_jar.sha512 @@ -1,14 +1,14 @@ -21cf5171c84fb12d0903d4f7d4f62e8b3dc60142ca1c717c46f7e09f6d40ea9a05a5bca34d468e00a00b427991de966fb060dcc282d532ee6a21567f802abfab classgraph-4.4.11.jar -7287b1ea3e18423d027a99ce40ae72e46e1700a65b474d2ec09af6a17b10653b7c2e69e9bb87efe14f4c593dc66b6370ea566fce90edb4b4190a903046817e6f ecj-3.13.102.jar -4626b970aa4d04422db93a4847eb9749768042255e0ba4d4944e1a8ca854de9e5eb093fbf5f0c05b122b65885324a9afdb1819acfb936514848c0726537cd403 jetty-http-9.4.12.v20180830.jar -19c0ea335efd54f6758b64725b4938cd124e60856b8966e1c60d33a5ebe3c62eea5babe5974c3a1b2c5ea49013ba5fc99aaa1a27e5a9c85e46f693fc679e5309 jetty-server-9.4.12.v20180830.jar -37ab6c29e925138d09a99bba9ead16b693318d8e098f1cf19fb56438c5d96479410628a84fa5a6c410850226acf1542aeab2a4894cebc9af8afa021c767d71f0 jetty-util-9.4.12.v20180830.jar -bf57568311fceb52f6611a2284d395cf181b634e4b44179766201f27b5a8c6981339aa35b3a0cb270d81d2a026e3ee724912040e0df5cf5ffd6442588b5b1e49 tomcat-api-9.0.13.jar -68238e5e00c7d0f0b159950a3c7ad6f666b343f7b31c5f0349a1e3184dea9c96ef50def82cc1025d6bcd4bc564d36306a169f96be7677185585ea2469b84a128 tomcat-el-api-9.0.13.jar -51c40eca728a34a96b2af0891355733e5ab5fc3ac5eec62f57e1ea905426b573bf9d55637d3b831838694054fc4d9dd06e48eb98eca57584d2750651bd286e0b tomcat-jasper-9.0.13.jar -ec64d3796a7f7224b451659f7f2b4a48da8a63da46557934d82c07420fa237a23c077f94908caa557ef51776d9e0a98b75cf43e6639c07f26aedb1ed4be99a62 tomcat-jasper-el-9.0.13.jar -2b5b92269c92e981268e346dc1484ebf4c7e481c2be26d90b02f650f94097bc8b9f9f1bc6579896e036a69747f41bf8493b3fa8d7626beaba95b28ae77f0f8ab tomcat-jsp-api-9.0.13.jar -f97891d80a6f96e9c10e5aa1d6a917961307f1a523266f4b0b857bb9a9bbe13ad09352a52e287d2dff84d18948ca25908ff13689eaea5bd806053c87822a892d tomcat-juli-9.0.13.jar -6579b30d95fa104663e2dced86115593868f18448f38235d88ccb003eeeeb61b49532300a30e7437e5c709891601f6b9909c33f1f7bcdf93ac118b67a742fdf1 tomcat-servlet-api-9.0.13.jar -af5d2f0209b7977e3d2ff6bb87dfa72aaa6684e9cd1aab7e48b4f0d389549c052c4d42cf3bd4c08837d79738609a9dfd93c9fdcf740d7a0ae851d9f8523ff65d tomcat-util-9.0.13.jar -18fb1a0f7e4ceb3416fc005e14f2eff6c49422806df03dfdd2a16ecc1292d55d59645707d408853573294446c2f525572dc3c94d45d32856c5d040945cbe416d tomcat-util-scan-9.0.13.jar +083f2102a50f2c3a4fc3993d328ddee17707b0529fdcee4868bd3b6901ff20a3d97e9733f83b2e46828469ecc273a1a3a5da26e4ba3d63f725a81b0b9e3e49e6 classgraph-4.8.90.jar +5e63e06eccd76a61249c7c6bd97e5b77dc37335bfdf316d941b00a16bae4f152d4306bac4e45c8422976187b0d77fe30cf7e6f6d4425f7732a3cb0cfa954385b ecj-3.23.0.jar +9abb4ba6802c39526b755429e3f41d641408f91f1c48844cd56cda116fb1479f05f1a3d90c18f30386b07d5593221c1f85ef565da0c6121b6ff005ec7d9c3bd9 jetty-http-9.4.33.v20201020.jar +e60e73ed3fd18e029c8e3cae04eac01d05f985dd0d9059eea620f4f9acac60799706d4b56b5a0f10cc33ff9272afdf11e712c604e31a77fa1afa1f95a57294ce jetty-server-9.4.33.v20201020.jar +35c5ad89d92e77f542d3ce42249f01c478706e2a013c5ab6a9504cf43520e3de90280d82a11389e9909060835421e3caada68e77d86df76a84856be60361d550 jetty-util-9.4.33.v20201020.jar +2d28d12196a7c2d342876d7e63c3a87fd613aadb32afad28d428d5dd9d9d0ebfa8300de3681d885455ac81d215326b11b845346416d5bd12d0e27c1763602caa tomcat-api-9.0.39.jar +661e613ae966e9b1fcc8815823c273fd7a05a6f8d3c469d86ce50650c1f350cbfb624fde8ee86235603078d9c5ccae5dfec8e03e10b5cb1e11109ae6ae4e85bb tomcat-el-api-9.0.39.jar +541d4794e1e15f26d7853cd7841f031e66fd3fc345102871cf58013591c8b57b6ab470d8ac28781c077f25e8d7fc06c19c6444db8e4a28ce4767ddbd907be004 tomcat-jasper-9.0.39.jar +57b916cb87f178604751f52f23f57fbd05da3001c9419602836e25d779727500bc1fceae195d5758aff28393b396751538f2ab7a9390738ebe4d0b4bd17b8ee4 tomcat-jasper-el-9.0.39.jar +d34deeecdecd59e40225ca3ad0ac197fc7f7845f09f551f5f17290ff9971741e62fac4bde16499d585b3d7c5574927c09740c0976dbddacecb19b4772f60c3e3 tomcat-jsp-api-9.0.39.jar +ee43f08f26e1d48c5cd3719e1eeec26e4ef389647a0bcd31435ed85c6c8770ab48fe287c60432aa8821b8fc8fce8ef0977a29b29c8c3b3cb246be53fb0182d1a tomcat-juli-9.0.39.jar +c387931fba85eae7ce030601dbd760cc7c2b0242a670b0c62321a8972e534076a40150044886053b3771099aa57c89057892d4f7fda84f65fa67b9c53dd51400 tomcat-servlet-api-9.0.39.jar +aa0b5ff10cdc686ba1dbc8a35d75aa42196126cbbde0d1ab1b9211ba57987a246bbe825c89a88104a0a60869fcbff4a8067bb247e42b69c887d6b48ee5c4a417 tomcat-util-9.0.39.jar +42c4b698cfb0a1fb154e290ac265c14693a9ef094e54116a324864f5b483f37d7a7decf67f7fe43b546cf1d2f25e9d763adc598a5e92979f756299e981dde43d tomcat-util-scan-9.0.39.jar diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py index fe0dde7a..b57d48bf 100644 --- a/test/unit/applications/lang/java.py +++ b/test/unit/applications/lang/java.py @@ -52,7 +52,7 @@ class TestApplicationJava(TestApplicationProto): os.makedirs(classes_path) classpath = ( - option.current_dir + '/build/tomcat-servlet-api-9.0.13.jar' + option.current_dir + '/build/tomcat-servlet-api-9.0.39.jar' ) ws_jars = glob.glob( -- cgit From 5182d2c398a07f65f2b2334bc04d79130a40808c Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Tue, 3 Nov 2020 01:31:05 +0000 Subject: Tests: force applications to build for JVM 8. This change is made to avoid situations when an application is compiled for a version newer than the Java module used. --- test/unit/applications/lang/java.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py index b57d48bf..e1bd5e0c 100644 --- a/test/unit/applications/lang/java.py +++ b/test/unit/applications/lang/java.py @@ -64,6 +64,7 @@ class TestApplicationJava(TestApplicationProto): javac = [ 'javac', + '-target', '8', '-source', '8', '-nowarn', '-encoding', 'utf-8', '-d', classes_path, '-classpath', classpath + ':' + ws_jars[0], -- cgit From a29d970f106ee10b57ab7ddf8fae4a02eac020b2 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Tue, 3 Nov 2020 01:32:40 +0000 Subject: Tests: disabled detailed output by default. --- test/pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pytest.ini b/test/pytest.ini index 0c3a330c..fe86cef2 100644 --- a/test/pytest.ini +++ b/test/pytest.ini @@ -1,3 +1,3 @@ [pytest] -addopts = -vvv -s --detailed --print_log +addopts = -vvv -s --print_log python_functions = test_* -- cgit From 4225361f0ea7d230c80209d76fbc67a932651380 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 5 Nov 2020 00:04:58 +0300 Subject: Python: introducting macro to simplify minor version check. --- src/python/nxt_python.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/python/nxt_python.h b/src/python/nxt_python.h index 3dd04e31..6d7eba8e 100644 --- a/src/python/nxt_python.h +++ b/src/python/nxt_python.h @@ -11,6 +11,8 @@ #include #include +#define NXT_PYTHON_VER(maj, min) ((maj << 24) | (min << 16)) + #if PY_MAJOR_VERSION == 3 #define NXT_PYTHON_BYTES_TYPE "bytestring" @@ -31,7 +33,7 @@ #define PyUnicode_GET_LENGTH PyUnicode_GET_SIZE #endif -#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 5 +#if PY_VERSION_HEX >= NXT_PYTHON_VER(3, 5) #define NXT_HAVE_ASGI 1 #endif -- cgit From 8dcb0b9987033d0349a6ecf528014a9daa574787 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 5 Nov 2020 00:04:59 +0300 Subject: Python: request processing in multiple threads. This closes #459 issue on GitHub. --- src/nxt_application.h | 2 + src/nxt_conf_validation.c | 58 ++++ src/nxt_main_process.c | 12 + src/python/nxt_python.c | 268 ++++++++++++++++-- src/python/nxt_python.h | 19 +- src/python/nxt_python_asgi.c | 490 ++++++++++++++++++++------------- src/python/nxt_python_asgi.h | 33 ++- src/python/nxt_python_asgi_http.c | 36 ++- src/python/nxt_python_asgi_lifespan.c | 102 +++---- src/python/nxt_python_asgi_str.c | 2 +- src/python/nxt_python_asgi_str.h | 2 +- src/python/nxt_python_asgi_websocket.c | 21 +- src/python/nxt_python_wsgi.c | 444 +++++++++++++++-------------- 13 files changed, 992 insertions(+), 497 deletions(-) diff --git a/src/nxt_application.h b/src/nxt_application.h index cb49a033..7f8ec1ba 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -51,6 +51,8 @@ typedef struct { nxt_str_t path; nxt_str_t module; char *callable; + uint32_t threads; + uint32_t thread_stack_size; } nxt_python_app_conf_t; diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index b44509e3..97a13e38 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -95,6 +95,10 @@ static nxt_int_t nxt_conf_vldt_return(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_proxy(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_thread_stack_size(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_routes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_routes_member(nxt_conf_validation_t *vldt, @@ -489,6 +493,14 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_python_members[] = { }, { .name = nxt_string("callable"), .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("threads"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_threads, + }, { + .name = nxt_string("thread_stack_size"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_thread_stack_size, }, NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) @@ -1328,6 +1340,52 @@ nxt_conf_vldt_proxy(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } +static nxt_int_t +nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, + void *data) +{ + int64_t threads; + + threads = nxt_conf_get_number(value); + + if (threads < 1) { + return nxt_conf_vldt_error(vldt, "The \"threads\" number must be " + "equal to or greater than 1."); + } + + if (threads > NXT_INT32_T_MAX) { + return nxt_conf_vldt_error(vldt, "The \"threads\" number must " + "not exceed %d.", NXT_INT32_T_MAX); + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_conf_vldt_thread_stack_size(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data) +{ + int64_t size; + + size = nxt_conf_get_number(value); + + if (size < PTHREAD_STACK_MIN) { + return nxt_conf_vldt_error(vldt, "The \"thread_stack_size\" number " + "must be equal to or greater than %d.", + PTHREAD_STACK_MIN); + } + + if ((size % nxt_pagesize) != 0) { + return nxt_conf_vldt_error(vldt, "The \"thread_stack_size\" number " + "must be a multiple of the system page size (%d).", + nxt_pagesize); + } + + return NXT_OK; +} + + static nxt_int_t nxt_conf_vldt_routes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index e4ec8b57..16c405ac 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -197,6 +197,18 @@ static nxt_conf_map_t nxt_python_app_conf[] = { NXT_CONF_MAP_CSTRZ, offsetof(nxt_common_app_conf_t, u.python.callable), }, + + { + nxt_string("threads"), + NXT_CONF_MAP_INT32, + offsetof(nxt_common_app_conf_t, u.python.threads), + }, + + { + nxt_string("thread_stack_size"), + NXT_CONF_MAP_INT32, + offsetof(nxt_common_app_conf_t, u.python.thread_stack_size), + }, }; diff --git a/src/python/nxt_python.c b/src/python/nxt_python.c index 01534a47..26a6f093 100644 --- a/src/python/nxt_python.c +++ b/src/python/nxt_python.c @@ -15,8 +15,20 @@ #include NXT_PYTHON_MOUNTS_H +typedef struct { + pthread_t thread; + nxt_unit_ctx_t *ctx; + void *ctx_data; +} nxt_py_thread_info_t; + + static nxt_int_t nxt_python_start(nxt_task_t *task, nxt_process_data_t *data); +static int nxt_python_init_threads(nxt_python_app_conf_t *c); +static int nxt_python_ready_handler(nxt_unit_ctx_t *ctx); +static void *nxt_python_thread_func(void *main_ctx); +static void nxt_python_join_threads(nxt_unit_ctx_t *ctx, + nxt_python_app_conf_t *c); static void nxt_python_atexit(void); static uint32_t compat[] = { @@ -44,11 +56,15 @@ static wchar_t *nxt_py_home; static char *nxt_py_home; #endif +static pthread_attr_t *nxt_py_thread_attr; +static nxt_py_thread_info_t *nxt_py_threads; +static nxt_python_proto_t nxt_py_proto; + static nxt_int_t nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) { - int rc, asgi; + int rc; char *nxt_py_module; size_t len; PyObject *obj, *pypath, *module; @@ -124,9 +140,17 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) Py_InitializeEx(0); +#if PY_VERSION_HEX < NXT_PYTHON_VER(3, 7) + if (c->threads > 1) { + PyEval_InitThreads(); + } +#endif + module = NULL; obj = NULL; + python_init.ctx_data = NULL; + obj = PySys_GetObject((char *) "stderr"); if (nxt_slow_path(obj == NULL)) { nxt_alert(task, "Python failed to get \"sys.stderr\" object"); @@ -216,35 +240,50 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) nxt_unit_default_init(task, &python_init); + python_init.data = c; python_init.shm_limit = data->app->shm_limit; + python_init.callbacks.ready_handler = nxt_python_ready_handler; - asgi = nxt_python_asgi_check(nxt_py_application); - - if (asgi) { - rc = nxt_python_asgi_init(task, &python_init); + if (nxt_python_asgi_check(nxt_py_application)) { + rc = nxt_python_asgi_init(&python_init, &nxt_py_proto); } else { - rc = nxt_python_wsgi_init(task, &python_init); + rc = nxt_python_wsgi_init(&python_init, &nxt_py_proto); } - if (nxt_slow_path(rc == NXT_ERROR)) { + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { goto fail; } + rc = nxt_py_proto.ctx_data_alloc(&python_init.ctx_data); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + goto fail; + } + + rc = nxt_python_init_threads(c); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + goto fail; + } + + if (nxt_py_proto.startup != NULL) { + if (nxt_py_proto.startup(python_init.ctx_data) != NXT_UNIT_OK) { + goto fail; + } + } + unit_ctx = nxt_unit_init(&python_init); if (nxt_slow_path(unit_ctx == NULL)) { goto fail; } - if (asgi) { - rc = nxt_python_asgi_run(unit_ctx); + rc = nxt_py_proto.run(unit_ctx); - } else { - rc = nxt_python_wsgi_run(unit_ctx); - } + nxt_python_join_threads(unit_ctx, c); nxt_unit_done(unit_ctx); + nxt_py_proto.ctx_data_free(python_init.ctx_data); + nxt_python_atexit(); exit(rc); @@ -253,6 +292,12 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) fail: + nxt_python_join_threads(NULL, c); + + if (python_init.ctx_data != NULL) { + nxt_py_proto.ctx_data_free(python_init.ctx_data); + } + Py_XDECREF(obj); Py_XDECREF(module); @@ -262,7 +307,195 @@ fail: } -nxt_int_t +static int +nxt_python_init_threads(nxt_python_app_conf_t *c) +{ + int res; + uint32_t i; + nxt_py_thread_info_t *ti; + static pthread_attr_t attr; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + if (c->thread_stack_size > 0) { + res = pthread_attr_init(&attr); + if (nxt_slow_path(res != 0)) { + nxt_unit_alert(NULL, "thread attr init failed: %s (%d)", + strerror(res), res); + + return NXT_UNIT_ERROR; + } + + res = pthread_attr_setstacksize(&attr, c->thread_stack_size); + if (nxt_slow_path(res != 0)) { + nxt_unit_alert(NULL, "thread attr set stack size failed: %s (%d)", + strerror(res), res); + + return NXT_UNIT_ERROR; + } + + nxt_py_thread_attr = &attr; + } + + nxt_py_threads = nxt_unit_malloc(NULL, sizeof(nxt_py_thread_info_t) + * (c->threads - 1)); + if (nxt_slow_path(nxt_py_threads == NULL)) { + nxt_unit_alert(NULL, "Failed to allocate thread info array"); + + return NXT_UNIT_ERROR; + } + + memset(nxt_py_threads, 0, sizeof(nxt_py_thread_info_t) * (c->threads - 1)); + + for (i = 0; i < c->threads - 1; i++) { + ti = &nxt_py_threads[i]; + + res = nxt_py_proto.ctx_data_alloc(&ti->ctx_data); + if (nxt_slow_path(res != NXT_UNIT_OK)) { + return NXT_UNIT_ERROR; + } + } + + return NXT_UNIT_OK; +} + + +static int +nxt_python_ready_handler(nxt_unit_ctx_t *ctx) +{ + int res; + uint32_t i; + nxt_py_thread_info_t *ti; + nxt_python_app_conf_t *c; + + if (nxt_py_proto.ready != NULL) { + res = nxt_py_proto.ready(ctx); + if (nxt_slow_path(res != NXT_UNIT_OK)) { + return NXT_UNIT_ERROR; + } + } + + /* Worker thread context. */ + if (!nxt_unit_is_main_ctx(ctx)) { + return NXT_UNIT_OK; + } + + c = ctx->unit->data; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + for (i = 0; i < c->threads - 1; i++) { + ti = &nxt_py_threads[i]; + + ti->ctx = ctx; + + res = pthread_create(&ti->thread, nxt_py_thread_attr, + nxt_python_thread_func, ti); + + if (nxt_fast_path(res == 0)) { + nxt_unit_debug(ctx, "thread #%d created", (int) (i + 1)); + + } else { + nxt_unit_alert(ctx, "thread #%d create failed: %s (%d)", + (int) (i + 1), strerror(res), res); + } + } + + return NXT_UNIT_OK; +} + + +static void * +nxt_python_thread_func(void *data) +{ + nxt_unit_ctx_t *ctx; + PyGILState_STATE gstate; + nxt_py_thread_info_t *ti; + + ti = data; + + nxt_unit_debug(ti->ctx, "worker thread #%d start", + (int) (ti - nxt_py_threads + 1)); + + gstate = PyGILState_Ensure(); + + if (nxt_py_proto.startup != NULL) { + if (nxt_py_proto.startup(ti->ctx_data) != NXT_UNIT_OK) { + goto fail; + } + } + + ctx = nxt_unit_ctx_alloc(ti->ctx, ti->ctx_data); + if (nxt_slow_path(ctx == NULL)) { + goto fail; + } + + (void) nxt_py_proto.run(ctx); + + nxt_unit_done(ctx); + +fail: + + PyGILState_Release(gstate); + + nxt_unit_debug(NULL, "worker thread #%d end", + (int) (ti - nxt_py_threads + 1)); + + return NULL; +} + + +static void +nxt_python_join_threads(nxt_unit_ctx_t *ctx, nxt_python_app_conf_t *c) +{ + int res; + uint32_t i; + PyThreadState *thread_state; + nxt_py_thread_info_t *ti; + + if (nxt_py_threads == NULL) { + return; + } + + thread_state = PyEval_SaveThread(); + + for (i = 0; i < c->threads - 1; i++) { + ti = &nxt_py_threads[i]; + + if ((uintptr_t) ti->thread == 0) { + continue; + } + + res = pthread_join(ti->thread, NULL); + + if (nxt_fast_path(res == 0)) { + nxt_unit_debug(ctx, "thread #%d joined", (int) (i + 1)); + + } else { + nxt_unit_alert(ctx, "thread #%d join failed: %s (%d)", + (int) (i + 1), strerror(res), res); + } + } + + PyEval_RestoreThread(thread_state); + + for (i = 0; i < c->threads - 1; i++) { + ti = &nxt_py_threads[i]; + + if (ti->ctx_data != NULL) { + nxt_py_proto.ctx_data_free(ti->ctx_data); + } + } + + nxt_unit_free(NULL, nxt_py_threads); +} + + +int nxt_python_init_strings(nxt_python_string_t *pstr) { PyObject *obj; @@ -271,7 +504,7 @@ nxt_python_init_strings(nxt_python_string_t *pstr) obj = PyString_FromStringAndSize((char *) pstr->string.start, pstr->string.length); if (nxt_slow_path(obj == NULL)) { - return NXT_ERROR; + return NXT_UNIT_ERROR; } PyUnicode_InternInPlace(&obj); @@ -281,7 +514,7 @@ nxt_python_init_strings(nxt_python_string_t *pstr) pstr++; } - return NXT_OK; + return NXT_UNIT_OK; } @@ -304,8 +537,9 @@ nxt_python_done_strings(nxt_python_string_t *pstr) static void nxt_python_atexit(void) { - nxt_python_wsgi_done(); - nxt_python_asgi_done(); + if (nxt_py_proto.done != NULL) { + nxt_py_proto.done(); + } Py_XDECREF(nxt_py_stderr_flush); Py_XDECREF(nxt_py_application); diff --git a/src/python/nxt_python.h b/src/python/nxt_python.h index 6d7eba8e..b581dd46 100644 --- a/src/python/nxt_python.h +++ b/src/python/nxt_python.h @@ -44,20 +44,25 @@ typedef struct { PyObject **object_p; } nxt_python_string_t; +typedef struct { + int (*ctx_data_alloc)(void **pdata); + void (*ctx_data_free)(void *data); + int (*startup)(void *data); + int (*run)(nxt_unit_ctx_t *ctx); + int (*ready)(nxt_unit_ctx_t *ctx); + void (*done)(void); +} nxt_python_proto_t; + -nxt_int_t nxt_python_init_strings(nxt_python_string_t *pstr); +int nxt_python_init_strings(nxt_python_string_t *pstr); void nxt_python_done_strings(nxt_python_string_t *pstr); void nxt_python_print_exception(void); -nxt_int_t nxt_python_wsgi_init(nxt_task_t *task, nxt_unit_init_t *init); -int nxt_python_wsgi_run(nxt_unit_ctx_t *ctx); -void nxt_python_wsgi_done(void); +int nxt_python_wsgi_init(nxt_unit_init_t *init, nxt_python_proto_t *proto); int nxt_python_asgi_check(PyObject *obj); -nxt_int_t nxt_python_asgi_init(nxt_task_t *task, nxt_unit_init_t *init); -nxt_int_t nxt_python_asgi_run(nxt_unit_ctx_t *ctx); -void nxt_python_asgi_done(void); +int nxt_python_asgi_init(nxt_unit_init_t *init, nxt_python_proto_t *proto); #endif /* _NXT_PYTHON_H_INCLUDED_ */ diff --git a/src/python/nxt_python_asgi.c b/src/python/nxt_python_asgi.c index 72408ea1..a188ff56 100644 --- a/src/python/nxt_python_asgi.c +++ b/src/python/nxt_python_asgi.c @@ -16,6 +16,13 @@ #include +static int nxt_python_asgi_ctx_data_alloc(void **pdata); +static void nxt_python_asgi_ctx_data_free(void *data); +static int nxt_python_asgi_startup(void *data); +static int nxt_python_asgi_run(nxt_unit_ctx_t *ctx); + +static void nxt_py_asgi_remove_reader(nxt_unit_ctx_t *ctx, + nxt_unit_port_t *port); static void nxt_py_asgi_request_handler(nxt_unit_request_info_t *req); static PyObject *nxt_py_asgi_create_http_scope(nxt_unit_request_info_t *req); @@ -24,30 +31,32 @@ static PyObject *nxt_py_asgi_create_address(nxt_unit_sptr_t *sptr, uint8_t len, static PyObject *nxt_py_asgi_create_header(nxt_unit_field_t *f); static PyObject *nxt_py_asgi_create_subprotocols(nxt_unit_field_t *f); +static int nxt_python_asgi_ready(nxt_unit_ctx_t *ctx); + static int nxt_py_asgi_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port); static void nxt_py_asgi_remove_port(nxt_unit_t *lib, nxt_unit_port_t *port); static void nxt_py_asgi_quit(nxt_unit_ctx_t *ctx); static void nxt_py_asgi_shm_ack_handler(nxt_unit_ctx_t *ctx); static PyObject *nxt_py_asgi_port_read(PyObject *self, PyObject *args); +static void nxt_python_asgi_done(void); -PyObject *nxt_py_loop_run_until_complete; -PyObject *nxt_py_loop_create_future; -PyObject *nxt_py_loop_create_task; - -nxt_queue_t nxt_py_asgi_drain_queue; - -static PyObject *nxt_py_loop_call_soon; -static PyObject *nxt_py_quit_future; -static PyObject *nxt_py_quit_future_set_result; -static PyObject *nxt_py_loop_add_reader; -static PyObject *nxt_py_loop_remove_reader; -static PyObject *nxt_py_port_read; +static PyObject *nxt_py_port_read; +static nxt_unit_port_t *nxt_py_shared_port; static PyMethodDef nxt_py_port_read_method = {"unit_port_read", nxt_py_asgi_port_read, METH_VARARGS, ""}; +static nxt_python_proto_t nxt_py_asgi_proto = { + .ctx_data_alloc = nxt_python_asgi_ctx_data_alloc, + .ctx_data_free = nxt_python_asgi_ctx_data_free, + .startup = nxt_python_asgi_startup, + .run = nxt_python_asgi_run, + .ready = nxt_python_asgi_ready, + .done = nxt_python_asgi_done, +}; + #define NXT_UNIT_HASH_WS_PROTOCOL 0xED0A @@ -102,202 +111,254 @@ nxt_python_asgi_check(PyObject *obj) } -nxt_int_t -nxt_python_asgi_init(nxt_task_t *task, nxt_unit_init_t *init) +int +nxt_python_asgi_init(nxt_unit_init_t *init, nxt_python_proto_t *proto) { - PyObject *asyncio, *loop, *get_event_loop; - nxt_int_t rc; - - nxt_debug(task, "asgi_init"); + nxt_unit_debug(NULL, "asgi_init"); - if (nxt_slow_path(nxt_py_asgi_str_init() != NXT_OK)) { - nxt_alert(task, "Python failed to init string objects"); - return NXT_ERROR; + if (nxt_slow_path(nxt_py_asgi_str_init() != NXT_UNIT_OK)) { + nxt_unit_alert(NULL, "Python failed to init string objects"); + return NXT_UNIT_ERROR; } - asyncio = PyImport_ImportModule("asyncio"); - if (nxt_slow_path(asyncio == NULL)) { - nxt_alert(task, "Python failed to import module 'asyncio'"); - nxt_python_print_exception(); - return NXT_ERROR; + nxt_py_port_read = PyCFunction_New(&nxt_py_port_read_method, NULL); + if (nxt_slow_path(nxt_py_port_read == NULL)) { + nxt_unit_alert(NULL, + "Python failed to initialize the 'port_read' function"); + return NXT_UNIT_ERROR; } - loop = NULL; - get_event_loop = PyDict_GetItemString(PyModule_GetDict(asyncio), - "get_event_loop"); - if (nxt_slow_path(get_event_loop == NULL)) { - nxt_alert(task, - "Python failed to get 'get_event_loop' from module 'asyncio'"); - goto fail; + if (nxt_slow_path(nxt_py_asgi_http_init() == NXT_UNIT_ERROR)) { + return NXT_UNIT_ERROR; } - if (nxt_slow_path(PyCallable_Check(get_event_loop) == 0)) { - nxt_alert(task, "'asyncio.get_event_loop' is not a callable object"); - goto fail; + if (nxt_slow_path(nxt_py_asgi_websocket_init() == NXT_UNIT_ERROR)) { + return NXT_UNIT_ERROR; } - loop = PyObject_CallObject(get_event_loop, NULL); - if (nxt_slow_path(loop == NULL)) { - nxt_alert(task, "Python failed to call 'asyncio.get_event_loop'"); - goto fail; - } + init->callbacks.request_handler = nxt_py_asgi_request_handler; + init->callbacks.data_handler = nxt_py_asgi_http_data_handler; + init->callbacks.websocket_handler = nxt_py_asgi_websocket_handler; + init->callbacks.close_handler = nxt_py_asgi_websocket_close_handler; + init->callbacks.quit = nxt_py_asgi_quit; + init->callbacks.shm_ack_handler = nxt_py_asgi_shm_ack_handler; + init->callbacks.add_port = nxt_py_asgi_add_port; + init->callbacks.remove_port = nxt_py_asgi_remove_port; - nxt_py_loop_create_task = PyObject_GetAttrString(loop, "create_task"); - if (nxt_slow_path(nxt_py_loop_create_task == NULL)) { - nxt_alert(task, "Python failed to get 'loop.create_task'"); - goto fail; - } + *proto = nxt_py_asgi_proto; - if (nxt_slow_path(PyCallable_Check(nxt_py_loop_create_task) == 0)) { - nxt_alert(task, "'loop.create_task' is not a callable object"); - goto fail; - } + return NXT_UNIT_OK; +} - nxt_py_loop_add_reader = PyObject_GetAttrString(loop, "add_reader"); - if (nxt_slow_path(nxt_py_loop_add_reader == NULL)) { - nxt_alert(task, "Python failed to get 'loop.add_reader'"); - goto fail; - } - if (nxt_slow_path(PyCallable_Check(nxt_py_loop_add_reader) == 0)) { - nxt_alert(task, "'loop.add_reader' is not a callable object"); - goto fail; - } +static int +nxt_python_asgi_ctx_data_alloc(void **pdata) +{ + uint32_t i; + PyObject *asyncio, *loop, *new_event_loop, *obj; + nxt_py_asgi_ctx_data_t *ctx_data; - nxt_py_loop_remove_reader = PyObject_GetAttrString(loop, "remove_reader"); - if (nxt_slow_path(nxt_py_loop_remove_reader == NULL)) { - nxt_alert(task, "Python failed to get 'loop.remove_reader'"); - goto fail; + ctx_data = nxt_unit_malloc(NULL, sizeof(nxt_py_asgi_ctx_data_t)); + if (nxt_slow_path(ctx_data == NULL)) { + nxt_unit_alert(NULL, "Failed to allocate context data"); + return NXT_UNIT_ERROR; } - if (nxt_slow_path(PyCallable_Check(nxt_py_loop_remove_reader) == 0)) { - nxt_alert(task, "'loop.remove_reader' is not a callable object"); - goto fail; - } + memset(ctx_data, 0, sizeof(nxt_py_asgi_ctx_data_t)); - nxt_py_loop_call_soon = PyObject_GetAttrString(loop, "call_soon"); - if (nxt_slow_path(nxt_py_loop_call_soon == NULL)) { - nxt_alert(task, "Python failed to get 'loop.call_soon'"); - goto fail; - } + nxt_queue_init(&ctx_data->drain_queue); - if (nxt_slow_path(PyCallable_Check(nxt_py_loop_call_soon) == 0)) { - nxt_alert(task, "'loop.call_soon' is not a callable object"); - goto fail; - } + struct { + const char *key; + PyObject **handler; - nxt_py_loop_run_until_complete = PyObject_GetAttrString(loop, - "run_until_complete"); - if (nxt_slow_path(nxt_py_loop_run_until_complete == NULL)) { - nxt_alert(task, "Python failed to get 'loop.run_until_complete'"); - goto fail; - } + } handlers[] = { + { "create_task", &ctx_data->loop_create_task }, + { "add_reader", &ctx_data->loop_add_reader }, + { "remove_reader", &ctx_data->loop_remove_reader }, + { "call_soon", &ctx_data->loop_call_soon }, + { "run_until_complete", &ctx_data->loop_run_until_complete }, + { "create_future", &ctx_data->loop_create_future }, + }; - if (nxt_slow_path(PyCallable_Check(nxt_py_loop_run_until_complete) == 0)) { - nxt_alert(task, "'loop.run_until_complete' is not a callable object"); - goto fail; - } + loop = NULL; - nxt_py_loop_create_future = PyObject_GetAttrString(loop, "create_future"); - if (nxt_slow_path(nxt_py_loop_create_future == NULL)) { - nxt_alert(task, "Python failed to get 'loop.create_future'"); + asyncio = PyImport_ImportModule("asyncio"); + if (nxt_slow_path(asyncio == NULL)) { + nxt_unit_alert(NULL, "Python failed to import module 'asyncio'"); + nxt_python_print_exception(); goto fail; } - if (nxt_slow_path(PyCallable_Check(nxt_py_loop_create_future) == 0)) { - nxt_alert(task, "'loop.create_future' is not a callable object"); + new_event_loop = PyDict_GetItemString(PyModule_GetDict(asyncio), + "new_event_loop"); + if (nxt_slow_path(new_event_loop == NULL)) { + nxt_unit_alert(NULL, + "Python failed to get 'new_event_loop' from module 'asyncio'"); goto fail; } - nxt_py_quit_future = PyObject_CallObject(nxt_py_loop_create_future, NULL); - if (nxt_slow_path(nxt_py_quit_future == NULL)) { - nxt_alert(task, "Python failed to create Future "); - nxt_python_print_exception(); + if (nxt_slow_path(PyCallable_Check(new_event_loop) == 0)) { + nxt_unit_alert(NULL, + "'asyncio.new_event_loop' is not a callable object"); goto fail; } - nxt_py_quit_future_set_result = PyObject_GetAttrString(nxt_py_quit_future, - "set_result"); - if (nxt_slow_path(nxt_py_quit_future_set_result == NULL)) { - nxt_alert(task, "Python failed to get 'future.set_result'"); + loop = PyObject_CallObject(new_event_loop, NULL); + if (nxt_slow_path(loop == NULL)) { + nxt_unit_alert(NULL, "Python failed to call 'asyncio.new_event_loop'"); goto fail; } - if (nxt_slow_path(PyCallable_Check(nxt_py_quit_future_set_result) == 0)) { - nxt_alert(task, "'future.set_result' is not a callable object"); - goto fail; + for (i = 0; i < nxt_nitems(handlers); i++) { + obj = PyObject_GetAttrString(loop, handlers[i].key); + if (nxt_slow_path(obj == NULL)) { + nxt_unit_alert(NULL, "Python failed to get 'loop.%s'", + handlers[i].key); + goto fail; + } + + *handlers[i].handler = obj; + + if (nxt_slow_path(PyCallable_Check(obj) == 0)) { + nxt_unit_alert(NULL, "'loop.%s' is not a callable object", + handlers[i].key); + goto fail; + } } - nxt_py_port_read = PyCFunction_New(&nxt_py_port_read_method, NULL); - if (nxt_slow_path(nxt_py_port_read == NULL)) { - nxt_alert(task, "Python failed to initialize the 'port_read' function"); + obj = PyObject_CallObject(ctx_data->loop_create_future, NULL); + if (nxt_slow_path(obj == NULL)) { + nxt_unit_alert(NULL, "Python failed to create Future "); + nxt_python_print_exception(); goto fail; } - nxt_queue_init(&nxt_py_asgi_drain_queue); + ctx_data->quit_future = obj; - if (nxt_slow_path(nxt_py_asgi_http_init(task) == NXT_ERROR)) { + obj = PyObject_GetAttrString(ctx_data->quit_future, "set_result"); + if (nxt_slow_path(obj == NULL)) { + nxt_unit_alert(NULL, "Python failed to get 'future.set_result'"); goto fail; } - if (nxt_slow_path(nxt_py_asgi_websocket_init(task) == NXT_ERROR)) { - goto fail; - } + ctx_data->quit_future_set_result = obj; - rc = nxt_py_asgi_lifespan_startup(task); - if (nxt_slow_path(rc == NXT_ERROR)) { + if (nxt_slow_path(PyCallable_Check(obj) == 0)) { + nxt_unit_alert(NULL, "'future.set_result' is not a callable object"); goto fail; } - init->callbacks.request_handler = nxt_py_asgi_request_handler; - init->callbacks.data_handler = nxt_py_asgi_http_data_handler; - init->callbacks.websocket_handler = nxt_py_asgi_websocket_handler; - init->callbacks.close_handler = nxt_py_asgi_websocket_close_handler; - init->callbacks.quit = nxt_py_asgi_quit; - init->callbacks.shm_ack_handler = nxt_py_asgi_shm_ack_handler; - init->callbacks.add_port = nxt_py_asgi_add_port; - init->callbacks.remove_port = nxt_py_asgi_remove_port; - Py_DECREF(loop); Py_DECREF(asyncio); - return NXT_OK; + *pdata = ctx_data; + + return NXT_UNIT_OK; fail: + nxt_python_asgi_ctx_data_free(ctx_data); + Py_XDECREF(loop); - Py_DECREF(asyncio); + Py_XDECREF(asyncio); + + return NXT_UNIT_ERROR; +} + - return NXT_ERROR; +static void +nxt_python_asgi_ctx_data_free(void *data) +{ + nxt_py_asgi_ctx_data_t *ctx_data; + + ctx_data = data; + + Py_XDECREF(ctx_data->loop_run_until_complete); + Py_XDECREF(ctx_data->loop_create_future); + Py_XDECREF(ctx_data->loop_create_task); + Py_XDECREF(ctx_data->loop_call_soon); + Py_XDECREF(ctx_data->loop_add_reader); + Py_XDECREF(ctx_data->loop_remove_reader); + Py_XDECREF(ctx_data->quit_future); + Py_XDECREF(ctx_data->quit_future_set_result); + + nxt_unit_free(NULL, ctx_data); } -nxt_int_t +static int +nxt_python_asgi_startup(void *data) +{ + return nxt_py_asgi_lifespan_startup(data); +} + + +static int nxt_python_asgi_run(nxt_unit_ctx_t *ctx) { - PyObject *res; + PyObject *res; + nxt_py_asgi_ctx_data_t *ctx_data; + + ctx_data = ctx->data; - res = PyObject_CallFunctionObjArgs(nxt_py_loop_run_until_complete, - nxt_py_quit_future, NULL); + res = PyObject_CallFunctionObjArgs(ctx_data->loop_run_until_complete, + ctx_data->quit_future, NULL); if (nxt_slow_path(res == NULL)) { nxt_unit_alert(ctx, "Python failed to call loop.run_until_complete"); nxt_python_print_exception(); - return NXT_ERROR; + return NXT_UNIT_ERROR; } Py_DECREF(res); - nxt_py_asgi_lifespan_shutdown(); + nxt_py_asgi_remove_reader(ctx, nxt_py_shared_port); + nxt_py_asgi_remove_reader(ctx, ctx_data->port); - return NXT_OK; + if (ctx_data->port != NULL) { + ctx_data->port->data = NULL; + ctx_data->port = NULL; + } + + nxt_py_asgi_lifespan_shutdown(ctx); + + return NXT_UNIT_OK; +} + + +static void +nxt_py_asgi_remove_reader(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) +{ + PyObject *res; + nxt_py_asgi_ctx_data_t *ctx_data; + + if (port == NULL || port->in_fd == -1) { + return; + } + + ctx_data = ctx->data; + + nxt_unit_debug(ctx, "asgi_remove_reader %d %p", port->in_fd, port); + + res = PyObject_CallFunctionObjArgs(ctx_data->loop_remove_reader, + PyLong_FromLong(port->in_fd), NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(ctx, "Python failed to remove_reader"); + nxt_python_print_exception(); + + return; + } + + Py_DECREF(res); } static void nxt_py_asgi_request_handler(nxt_unit_request_info_t *req) { - PyObject *scope, *res, *task, *receive, *send, *done, *asgi; + PyObject *scope, *res, *task, *receive, *send, *done, *asgi; + nxt_py_asgi_ctx_data_t *ctx_data; if (req->request->websocket_handshake) { asgi = nxt_py_asgi_websocket_create(req); @@ -365,7 +426,9 @@ nxt_py_asgi_request_handler(nxt_unit_request_info_t *req) goto release_scope; } - task = PyObject_CallFunctionObjArgs(nxt_py_loop_create_task, res, NULL); + ctx_data = req->ctx->data; + + task = PyObject_CallFunctionObjArgs(ctx_data->loop_create_task, res, NULL); if (nxt_slow_path(task == NULL)) { nxt_unit_req_error(req, "Python failed to call the create_task"); nxt_python_print_exception(); @@ -723,11 +786,47 @@ fail: } +static int +nxt_python_asgi_ready(nxt_unit_ctx_t *ctx) +{ + PyObject *res; + nxt_unit_port_t *port; + nxt_py_asgi_ctx_data_t *ctx_data; + + if (nxt_slow_path(nxt_py_shared_port == NULL)) { + return NXT_UNIT_ERROR; + } + + port = nxt_py_shared_port; + + nxt_unit_debug(ctx, "asgi_ready %d %p %p", port->in_fd, ctx, port); + + ctx_data = ctx->data; + + res = PyObject_CallFunctionObjArgs(ctx_data->loop_add_reader, + PyLong_FromLong(port->in_fd), + nxt_py_port_read, + PyLong_FromVoidPtr(ctx), + PyLong_FromVoidPtr(port), NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(ctx, "Python failed to add_reader"); + nxt_python_print_exception(); + + return NXT_UNIT_ERROR; + } + + Py_DECREF(res); + + return NXT_UNIT_OK; +} + + static int nxt_py_asgi_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) { - int nb; - PyObject *res; + int nb; + PyObject *res; + nxt_py_asgi_ctx_data_t *ctx_data; if (port->in_fd == -1) { return NXT_UNIT_OK; @@ -744,13 +843,25 @@ nxt_py_asgi_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) nxt_unit_debug(ctx, "asgi_add_port %d %p %p", port->in_fd, ctx, port); - res = PyObject_CallFunctionObjArgs(nxt_py_loop_add_reader, + if (port->id.id == NXT_UNIT_SHARED_PORT_ID) { + nxt_py_shared_port = port; + + return NXT_UNIT_OK; + } + + ctx_data = ctx->data; + + ctx_data->port = port; + port->data = ctx_data; + + res = PyObject_CallFunctionObjArgs(ctx_data->loop_add_reader, PyLong_FromLong(port->in_fd), nxt_py_port_read, PyLong_FromVoidPtr(ctx), PyLong_FromVoidPtr(port), NULL); if (nxt_slow_path(res == NULL)) { nxt_unit_alert(ctx, "Python failed to add_reader"); + nxt_python_print_exception(); return NXT_UNIT_ERROR; } @@ -764,53 +875,67 @@ nxt_py_asgi_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) static void nxt_py_asgi_remove_port(nxt_unit_t *lib, nxt_unit_port_t *port) { - PyObject *res; - - nxt_unit_debug(NULL, "asgi_remove_port %d %p", port->in_fd, port); - if (port->in_fd == -1) { return; } - res = PyObject_CallFunctionObjArgs(nxt_py_loop_remove_reader, - PyLong_FromLong(port->in_fd), NULL); - if (nxt_slow_path(res == NULL)) { - nxt_unit_alert(NULL, "Python failed to remove_reader"); - } + nxt_unit_debug(NULL, "asgi_remove_port %d %p", port->in_fd, port); - Py_DECREF(res); + if (nxt_py_shared_port == port) { + nxt_py_shared_port = NULL; + } } static void nxt_py_asgi_quit(nxt_unit_ctx_t *ctx) { - PyObject *res; + PyObject *res; + nxt_py_asgi_ctx_data_t *ctx_data; nxt_unit_debug(ctx, "asgi_quit %p", ctx); - res = PyObject_CallFunctionObjArgs(nxt_py_quit_future_set_result, + ctx_data = ctx->data; + + if (nxt_py_shared_port != NULL) { + res = PyObject_CallFunctionObjArgs(ctx_data->loop_remove_reader, + PyLong_FromLong(nxt_py_shared_port->in_fd), NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(NULL, "Python failed to remove_reader"); + nxt_python_print_exception(); + + } else { + Py_DECREF(res); + } + } + + res = PyObject_CallFunctionObjArgs(ctx_data->quit_future_set_result, PyLong_FromLong(0), NULL); if (nxt_slow_path(res == NULL)) { nxt_unit_alert(ctx, "Python failed to set_result"); - } + nxt_python_print_exception(); - Py_DECREF(res); + } else { + Py_DECREF(res); + } } static void nxt_py_asgi_shm_ack_handler(nxt_unit_ctx_t *ctx) { - int rc; - nxt_queue_link_t *lnk; + int rc; + nxt_queue_link_t *lnk; + nxt_py_asgi_ctx_data_t *ctx_data; + + ctx_data = ctx->data; - while (!nxt_queue_is_empty(&nxt_py_asgi_drain_queue)) { - lnk = nxt_queue_first(&nxt_py_asgi_drain_queue); + while (!nxt_queue_is_empty(&ctx_data->drain_queue)) { + lnk = nxt_queue_first(&ctx_data->drain_queue); rc = nxt_py_asgi_http_drain(lnk); if (rc == NXT_UNIT_AGAIN) { - break; + return; } nxt_queue_remove(lnk); @@ -859,7 +984,7 @@ nxt_py_asgi_port_read(PyObject *self, PyObject *args) if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { return PyErr_Format(PyExc_RuntimeError, - "error processing port message"); + "error processing port %d message", port->id.id); } Py_RETURN_NONE; @@ -996,8 +1121,8 @@ nxt_py_asgi_add_field(void *data, int i, PyObject *name, PyObject *val) PyObject * -nxt_py_asgi_set_result_soon(nxt_unit_request_info_t *req, PyObject *future, - PyObject *result) +nxt_py_asgi_set_result_soon(nxt_unit_request_info_t *req, + nxt_py_asgi_ctx_data_t *ctx_data, PyObject *future, PyObject *result) { PyObject *set_result, *res; @@ -1013,7 +1138,7 @@ nxt_py_asgi_set_result_soon(nxt_unit_request_info_t *req, PyObject *future, Py_CLEAR(future); - goto cleanup; + goto cleanup_result; } if (nxt_slow_path(PyCallable_Check(set_result) == 0)) { @@ -1024,7 +1149,7 @@ nxt_py_asgi_set_result_soon(nxt_unit_request_info_t *req, PyObject *future, goto cleanup; } - res = PyObject_CallFunctionObjArgs(nxt_py_loop_call_soon, set_result, + res = PyObject_CallFunctionObjArgs(ctx_data->loop_call_soon, set_result, result, NULL); if (nxt_slow_path(res == NULL)) { nxt_unit_req_alert(req, "Python failed to call 'loop.call_soon'"); @@ -1038,6 +1163,9 @@ nxt_py_asgi_set_result_soon(nxt_unit_request_info_t *req, PyObject *future, cleanup: Py_DECREF(set_result); + +cleanup_result: + Py_DECREF(result); return future; @@ -1147,6 +1275,17 @@ nxt_py_asgi_new_scope(nxt_unit_request_info_t *req, PyObject *type, } +void +nxt_py_asgi_drain_wait(nxt_unit_request_info_t *req, nxt_queue_link_t *link) +{ + nxt_py_asgi_ctx_data_t *ctx_data; + + ctx_data = req->ctx->data; + + nxt_queue_insert_tail(&ctx_data->drain_queue, link); +} + + void nxt_py_asgi_dealloc(PyObject *self) { @@ -1177,19 +1316,11 @@ nxt_py_asgi_next(PyObject *self) } -void +static void nxt_python_asgi_done(void) { nxt_py_asgi_str_done(); - Py_XDECREF(nxt_py_quit_future); - Py_XDECREF(nxt_py_quit_future_set_result); - Py_XDECREF(nxt_py_loop_run_until_complete); - Py_XDECREF(nxt_py_loop_create_future); - Py_XDECREF(nxt_py_loop_create_task); - Py_XDECREF(nxt_py_loop_call_soon); - Py_XDECREF(nxt_py_loop_add_reader); - Py_XDECREF(nxt_py_loop_remove_reader); Py_XDECREF(nxt_py_port_read); } @@ -1203,25 +1334,12 @@ nxt_python_asgi_check(PyObject *obj) } -nxt_int_t -nxt_python_asgi_init(nxt_task_t *task, nxt_unit_init_t *init) -{ - nxt_alert(task, "ASGI not implemented"); - return NXT_ERROR; -} - - -nxt_int_t -nxt_python_asgi_run(nxt_unit_ctx_t *ctx) +int +nxt_python_asgi_init(nxt_unit_init_t *init, nxt_python_proto_t *proto) { - nxt_unit_alert(ctx, "ASGI not implemented"); - return NXT_ERROR; + nxt_unit_alert(NULL, "ASGI not implemented"); + return NXT_UNIT_ERROR; } -void -nxt_python_asgi_done(void) -{ -} - #endif /* NXT_HAVE_ASGI */ diff --git a/src/python/nxt_python_asgi.h b/src/python/nxt_python_asgi.h index 24337c37..69d58477 100644 --- a/src/python/nxt_python_asgi.h +++ b/src/python/nxt_python_asgi.h @@ -10,6 +10,9 @@ typedef PyObject * (*nxt_py_asgi_enum_header_cb)(void *ctx, int i, PyObject *name, PyObject *val); +void nxt_py_asgi_drain_wait(nxt_unit_request_info_t *req, + nxt_queue_link_t *link); + typedef struct { uint32_t fields_count; uint32_t fields_size; @@ -20,6 +23,20 @@ typedef struct { uint64_t content_length; } nxt_py_asgi_add_field_ctx_t; +typedef struct { + nxt_queue_t drain_queue; + PyObject *loop_run_until_complete; + PyObject *loop_create_future; + PyObject *loop_create_task; + PyObject *loop_call_soon; + PyObject *loop_add_reader; + PyObject *loop_remove_reader; + PyObject *quit_future; + PyObject *quit_future_set_result; + PyObject *lifespan; + nxt_unit_port_t *port; +} nxt_py_asgi_ctx_data_t; + PyObject *nxt_py_asgi_enum_headers(PyObject *headers, nxt_py_asgi_enum_header_cb cb, void *data); @@ -27,7 +44,7 @@ PyObject *nxt_py_asgi_calc_size(void *data, int i, PyObject *n, PyObject *v); PyObject *nxt_py_asgi_add_field(void *data, int i, PyObject *n, PyObject *v); PyObject *nxt_py_asgi_set_result_soon(nxt_unit_request_info_t *req, - PyObject *future, PyObject *result); + nxt_py_asgi_ctx_data_t *ctx_data, PyObject *future, PyObject *result); PyObject *nxt_py_asgi_new_msg(nxt_unit_request_info_t *req, PyObject *type); PyObject *nxt_py_asgi_new_scope(nxt_unit_request_info_t *req, PyObject *type, PyObject *spec_version); @@ -37,24 +54,18 @@ PyObject *nxt_py_asgi_await(PyObject *self); PyObject *nxt_py_asgi_iter(PyObject *self); PyObject *nxt_py_asgi_next(PyObject *self); -nxt_int_t nxt_py_asgi_http_init(nxt_task_t *task); +int nxt_py_asgi_http_init(void); PyObject *nxt_py_asgi_http_create(nxt_unit_request_info_t *req); void nxt_py_asgi_http_data_handler(nxt_unit_request_info_t *req); int nxt_py_asgi_http_drain(nxt_queue_link_t *lnk); -nxt_int_t nxt_py_asgi_websocket_init(nxt_task_t *task); +int nxt_py_asgi_websocket_init(void); PyObject *nxt_py_asgi_websocket_create(nxt_unit_request_info_t *req); void nxt_py_asgi_websocket_handler(nxt_unit_websocket_frame_t *ws); void nxt_py_asgi_websocket_close_handler(nxt_unit_request_info_t *req); -nxt_int_t nxt_py_asgi_lifespan_startup(nxt_task_t *task); -nxt_int_t nxt_py_asgi_lifespan_shutdown(void); - -extern PyObject *nxt_py_loop_run_until_complete; -extern PyObject *nxt_py_loop_create_future; -extern PyObject *nxt_py_loop_create_task; - -extern nxt_queue_t nxt_py_asgi_drain_queue; +int nxt_py_asgi_lifespan_startup(nxt_py_asgi_ctx_data_t *ctx_data); +int nxt_py_asgi_lifespan_shutdown(nxt_unit_ctx_t *ctx); #endif /* _NXT_PYTHON_ASGI_H_INCLUDED_ */ diff --git a/src/python/nxt_python_asgi_http.c b/src/python/nxt_python_asgi_http.c index b07d61d6..a5034ea6 100644 --- a/src/python/nxt_python_asgi_http.c +++ b/src/python/nxt_python_asgi_http.c @@ -67,15 +67,16 @@ static PyTypeObject nxt_py_asgi_http_type = { static Py_ssize_t nxt_py_asgi_http_body_buf_size = 32 * 1024 * 1024; -nxt_int_t -nxt_py_asgi_http_init(nxt_task_t *task) +int +nxt_py_asgi_http_init(void) { if (nxt_slow_path(PyType_Ready(&nxt_py_asgi_http_type) != 0)) { - nxt_alert(task, "Python failed to initialize the 'http' type object"); - return NXT_ERROR; + nxt_unit_alert(NULL, + "Python failed to initialize the 'http' type object"); + return NXT_UNIT_ERROR; } - return NXT_OK; + return NXT_UNIT_OK; } @@ -106,6 +107,7 @@ nxt_py_asgi_http_receive(PyObject *self, PyObject *none) { PyObject *msg, *future; nxt_py_asgi_http_t *http; + nxt_py_asgi_ctx_data_t *ctx_data; nxt_unit_request_info_t *req; http = (nxt_py_asgi_http_t *) self; @@ -118,7 +120,9 @@ nxt_py_asgi_http_receive(PyObject *self, PyObject *none) return NULL; } - future = PyObject_CallObject(nxt_py_loop_create_future, NULL); + ctx_data = req->ctx->data; + + future = PyObject_CallObject(ctx_data->loop_create_future, NULL); if (nxt_slow_path(future == NULL)) { nxt_unit_req_alert(req, "Python failed to create Future object"); nxt_python_print_exception(); @@ -130,7 +134,7 @@ nxt_py_asgi_http_receive(PyObject *self, PyObject *none) } if (msg != Py_None) { - return nxt_py_asgi_set_result_soon(req, future, msg); + return nxt_py_asgi_set_result_soon(req, ctx_data, future, msg); } http->receive_future = future; @@ -329,11 +333,12 @@ nxt_py_asgi_http_response_start(nxt_py_asgi_http_t *http, PyObject *dict) static PyObject * nxt_py_asgi_http_response_body(nxt_py_asgi_http_t *http, PyObject *dict) { - int rc; - char *body_str; - ssize_t sent; - PyObject *body, *more_body, *future; - Py_ssize_t body_len, body_off; + int rc; + char *body_str; + ssize_t sent; + PyObject *body, *more_body, *future; + Py_ssize_t body_len, body_off; + nxt_py_asgi_ctx_data_t *ctx_data; body = PyDict_GetItem(dict, nxt_py_body_str); if (nxt_slow_path(body != NULL && !PyBytes_Check(body))) { @@ -371,6 +376,8 @@ nxt_py_asgi_http_response_body(nxt_py_asgi_http_t *http, PyObject *dict) body_off = 0; + ctx_data = http->req->ctx->data; + while (body_len > 0) { sent = nxt_unit_response_write_nb(http->req, body_str, body_len, 0); if (nxt_slow_path(sent < 0)) { @@ -382,7 +389,8 @@ nxt_py_asgi_http_response_body(nxt_py_asgi_http_t *http, PyObject *dict) "out of shared memory, %d", (int) body_len); - future = PyObject_CallObject(nxt_py_loop_create_future, NULL); + future = PyObject_CallObject(ctx_data->loop_create_future, + NULL); if (nxt_slow_path(future == NULL)) { nxt_unit_req_alert(http->req, "Python failed to create Future object"); @@ -396,7 +404,7 @@ nxt_py_asgi_http_response_body(nxt_py_asgi_http_t *http, PyObject *dict) Py_INCREF(http->send_body); http->send_body_off = body_off; - nxt_queue_insert_tail(&nxt_py_asgi_drain_queue, &http->link); + nxt_py_asgi_drain_wait(http->req, &http->link); http->send_future = future; Py_INCREF(http->send_future); diff --git a/src/python/nxt_python_asgi_lifespan.c b/src/python/nxt_python_asgi_lifespan.c index 14d0ee97..6910d2e8 100644 --- a/src/python/nxt_python_asgi_lifespan.c +++ b/src/python/nxt_python_asgi_lifespan.c @@ -15,15 +15,16 @@ typedef struct { PyObject_HEAD - int disabled; - int startup_received; - int startup_sent; - int shutdown_received; - int shutdown_sent; - int shutdown_called; - PyObject *startup_future; - PyObject *shutdown_future; - PyObject *receive_future; + nxt_py_asgi_ctx_data_t *ctx_data; + int disabled; + int startup_received; + int startup_sent; + int shutdown_received; + int shutdown_sent; + int shutdown_called; + PyObject *startup_future; + PyObject *shutdown_future; + PyObject *receive_future; } nxt_py_asgi_lifespan_t; @@ -39,8 +40,6 @@ static PyObject *nxt_py_asgi_lifespan_disable(nxt_py_asgi_lifespan_t *lifespan); static PyObject *nxt_py_asgi_lifespan_done(PyObject *self, PyObject *future); -static nxt_py_asgi_lifespan_t *nxt_py_lifespan; - static PyMethodDef nxt_py_asgi_lifespan_methods[] = { { "receive", nxt_py_asgi_lifespan_receive, METH_NOARGS, 0 }, { "send", nxt_py_asgi_lifespan_send, METH_O, 0 }, @@ -67,46 +66,46 @@ static PyTypeObject nxt_py_asgi_lifespan_type = { }; -nxt_int_t -nxt_py_asgi_lifespan_startup(nxt_task_t *task) +int +nxt_py_asgi_lifespan_startup(nxt_py_asgi_ctx_data_t *ctx_data) { + int rc; PyObject *scope, *res, *py_task, *receive, *send, *done; - nxt_int_t rc; nxt_py_asgi_lifespan_t *lifespan; if (nxt_slow_path(PyType_Ready(&nxt_py_asgi_lifespan_type) != 0)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to initialize the 'asgi_lifespan' type object"); - return NXT_ERROR; + return NXT_UNIT_ERROR; } lifespan = PyObject_New(nxt_py_asgi_lifespan_t, &nxt_py_asgi_lifespan_type); if (nxt_slow_path(lifespan == NULL)) { - nxt_alert(task, "Python failed to create lifespan object"); - return NXT_ERROR; + nxt_unit_alert(NULL, "Python failed to create lifespan object"); + return NXT_UNIT_ERROR; } - rc = NXT_ERROR; + rc = NXT_UNIT_ERROR; receive = PyObject_GetAttrString((PyObject *) lifespan, "receive"); if (nxt_slow_path(receive == NULL)) { - nxt_alert(task, "Python failed to get 'receive' method"); + nxt_unit_alert(NULL, "Python failed to get 'receive' method"); goto release_lifespan; } send = PyObject_GetAttrString((PyObject *) lifespan, "send"); if (nxt_slow_path(receive == NULL)) { - nxt_alert(task, "Python failed to get 'send' method"); + nxt_unit_alert(NULL, "Python failed to get 'send' method"); goto release_receive; } done = PyObject_GetAttrString((PyObject *) lifespan, "_done"); if (nxt_slow_path(receive == NULL)) { - nxt_alert(task, "Python failed to get '_done' method"); + nxt_unit_alert(NULL, "Python failed to get '_done' method"); goto release_send; } - lifespan->startup_future = PyObject_CallObject(nxt_py_loop_create_future, + lifespan->startup_future = PyObject_CallObject(ctx_data->loop_create_future, NULL); if (nxt_slow_path(lifespan->startup_future == NULL)) { nxt_unit_alert(NULL, "Python failed to create Future object"); @@ -115,6 +114,7 @@ nxt_py_asgi_lifespan_startup(nxt_task_t *task) goto release_done; } + lifespan->ctx_data = ctx_data; lifespan->disabled = 0; lifespan->startup_received = 0; lifespan->startup_sent = 0; @@ -132,21 +132,20 @@ nxt_py_asgi_lifespan_startup(nxt_task_t *task) res = PyObject_CallFunctionObjArgs(nxt_py_application, scope, receive, send, NULL); if (nxt_slow_path(res == NULL)) { - nxt_log(task, NXT_LOG_ERR, "Python failed to call the application"); + nxt_unit_error(NULL, "Python failed to call the application"); nxt_python_print_exception(); goto release_scope; } if (nxt_slow_path(!PyCoro_CheckExact(res))) { - nxt_log(task, NXT_LOG_ERR, - "Application result type is not a coroutine"); + nxt_unit_error(NULL, "Application result type is not a coroutine"); Py_DECREF(res); goto release_scope; } - py_task = PyObject_CallFunctionObjArgs(nxt_py_loop_create_task, res, NULL); + py_task = PyObject_CallFunctionObjArgs(ctx_data->loop_create_task, res, NULL); if (nxt_slow_path(py_task == NULL)) { - nxt_log(task, NXT_LOG_ERR, "Python failed to call the create_task"); + nxt_unit_alert(NULL, "Python failed to call the create_task"); nxt_python_print_exception(); Py_DECREF(res); goto release_scope; @@ -157,18 +156,17 @@ nxt_py_asgi_lifespan_startup(nxt_task_t *task) res = PyObject_CallMethodObjArgs(py_task, nxt_py_add_done_callback_str, done, NULL); if (nxt_slow_path(res == NULL)) { - nxt_log(task, NXT_LOG_ERR, - "Python failed to call 'task.add_done_callback'"); + nxt_unit_alert(NULL, "Python failed to call 'task.add_done_callback'"); nxt_python_print_exception(); goto release_task; } Py_DECREF(res); - res = PyObject_CallFunctionObjArgs(nxt_py_loop_run_until_complete, + res = PyObject_CallFunctionObjArgs(ctx_data->loop_run_until_complete, lifespan->startup_future, NULL); if (nxt_slow_path(res == NULL)) { - nxt_alert(task, "Python failed to call loop.run_until_complete"); + nxt_unit_alert(NULL, "Python failed to call loop.run_until_complete"); nxt_python_print_exception(); goto release_task; } @@ -176,10 +174,10 @@ nxt_py_asgi_lifespan_startup(nxt_task_t *task) Py_DECREF(res); if (lifespan->startup_sent == 1 || lifespan->disabled) { - nxt_py_lifespan = lifespan; - Py_INCREF(nxt_py_lifespan); + ctx_data->lifespan = (PyObject *) lifespan; + Py_INCREF(ctx_data->lifespan); - rc = NXT_OK; + rc = NXT_UNIT_OK; } release_task: @@ -201,17 +199,21 @@ release_lifespan: } -nxt_int_t -nxt_py_asgi_lifespan_shutdown(void) +int +nxt_py_asgi_lifespan_shutdown(nxt_unit_ctx_t *ctx) { PyObject *msg, *future, *res; nxt_py_asgi_lifespan_t *lifespan; + nxt_py_asgi_ctx_data_t *ctx_data; + + ctx_data = ctx->data; + + lifespan = (nxt_py_asgi_lifespan_t *) ctx_data->lifespan; - if (nxt_slow_path(nxt_py_lifespan == NULL || nxt_py_lifespan->disabled)) { - return NXT_OK; + if (nxt_slow_path(lifespan == NULL || lifespan->disabled)) { + return NXT_UNIT_OK; } - lifespan = nxt_py_lifespan; lifespan->shutdown_called = 1; if (lifespan->receive_future != NULL) { @@ -231,29 +233,29 @@ nxt_py_asgi_lifespan_shutdown(void) } if (lifespan->shutdown_sent) { - return NXT_OK; + return NXT_UNIT_OK; } - lifespan->shutdown_future = PyObject_CallObject(nxt_py_loop_create_future, + lifespan->shutdown_future = PyObject_CallObject(ctx_data->loop_create_future, NULL); if (nxt_slow_path(lifespan->shutdown_future == NULL)) { nxt_unit_alert(NULL, "Python failed to create Future object"); nxt_python_print_exception(); - return NXT_ERROR; + return NXT_UNIT_ERROR; } - res = PyObject_CallFunctionObjArgs(nxt_py_loop_run_until_complete, + res = PyObject_CallFunctionObjArgs(ctx_data->loop_run_until_complete, lifespan->shutdown_future, NULL); if (nxt_slow_path(res == NULL)) { nxt_unit_alert(NULL, "Python failed to call loop.run_until_complete"); nxt_python_print_exception(); - return NXT_ERROR; + return NXT_UNIT_ERROR; } Py_DECREF(res); Py_CLEAR(lifespan->shutdown_future); - return NXT_OK; + return NXT_UNIT_OK; } @@ -262,12 +264,14 @@ nxt_py_asgi_lifespan_receive(PyObject *self, PyObject *none) { PyObject *msg, *future; nxt_py_asgi_lifespan_t *lifespan; + nxt_py_asgi_ctx_data_t *ctx_data; lifespan = (nxt_py_asgi_lifespan_t *) self; + ctx_data = lifespan->ctx_data; nxt_unit_debug(NULL, "asgi_lifespan_receive"); - future = PyObject_CallObject(nxt_py_loop_create_future, NULL); + future = PyObject_CallObject(ctx_data->loop_create_future, NULL); if (nxt_slow_path(future == NULL)) { nxt_unit_alert(NULL, "Python failed to create Future object"); nxt_python_print_exception(); @@ -281,7 +285,7 @@ nxt_py_asgi_lifespan_receive(PyObject *self, PyObject *none) msg = nxt_py_asgi_new_msg(NULL, nxt_py_lifespan_startup_str); - return nxt_py_asgi_set_result_soon(NULL, future, msg); + return nxt_py_asgi_set_result_soon(NULL, ctx_data, future, msg); } if (lifespan->shutdown_called && !lifespan->shutdown_received) { @@ -289,7 +293,7 @@ nxt_py_asgi_lifespan_receive(PyObject *self, PyObject *none) msg = nxt_py_asgi_new_msg(NULL, nxt_py_lifespan_shutdown_str); - return nxt_py_asgi_set_result_soon(NULL, future, msg); + return nxt_py_asgi_set_result_soon(NULL, ctx_data, future, msg); } Py_INCREF(future); diff --git a/src/python/nxt_python_asgi_str.c b/src/python/nxt_python_asgi_str.c index 37fa7f04..34422973 100644 --- a/src/python/nxt_python_asgi_str.c +++ b/src/python/nxt_python_asgi_str.c @@ -124,7 +124,7 @@ static nxt_python_string_t nxt_py_asgi_strings[] = { }; -nxt_int_t +int nxt_py_asgi_str_init(void) { return nxt_python_init_strings(nxt_py_asgi_strings); diff --git a/src/python/nxt_python_asgi_str.h b/src/python/nxt_python_asgi_str.h index 3f389c62..92969fd2 100644 --- a/src/python/nxt_python_asgi_str.h +++ b/src/python/nxt_python_asgi_str.h @@ -62,7 +62,7 @@ extern PyObject *nxt_py_ws_str; extern PyObject *nxt_py_wss_str; -nxt_int_t nxt_py_asgi_str_init(void); +int nxt_py_asgi_str_init(void); void nxt_py_asgi_str_done(void); diff --git a/src/python/nxt_python_asgi_websocket.c b/src/python/nxt_python_asgi_websocket.c index 5a27b588..fc7d9fa4 100644 --- a/src/python/nxt_python_asgi_websocket.c +++ b/src/python/nxt_python_asgi_websocket.c @@ -98,16 +98,16 @@ static uint64_t nxt_py_asgi_ws_max_frame_size = 1024 * 1024; static uint64_t nxt_py_asgi_ws_max_buffer_size = 10 * 1024 * 1024; -nxt_int_t -nxt_py_asgi_websocket_init(nxt_task_t *task) +int +nxt_py_asgi_websocket_init(void) { if (nxt_slow_path(PyType_Ready(&nxt_py_asgi_websocket_type) != 0)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to initialize the \"asgi_websocket\" type object"); - return NXT_ERROR; + return NXT_UNIT_ERROR; } - return NXT_OK; + return NXT_UNIT_OK; } @@ -137,6 +137,7 @@ static PyObject * nxt_py_asgi_websocket_receive(PyObject *self, PyObject *none) { PyObject *future, *msg; + nxt_py_asgi_ctx_data_t *ctx_data; nxt_py_asgi_websocket_t *ws; ws = (nxt_py_asgi_websocket_t *) self; @@ -160,7 +161,9 @@ nxt_py_asgi_websocket_receive(PyObject *self, PyObject *none) "WebSocket already closed"); } - future = PyObject_CallObject(nxt_py_loop_create_future, NULL); + ctx_data = ws->req->ctx->data; + + future = PyObject_CallObject(ctx_data->loop_create_future, NULL); if (nxt_slow_path(future == NULL)) { nxt_unit_req_alert(ws->req, "Python failed to create Future object"); nxt_python_print_exception(); @@ -174,19 +177,19 @@ nxt_py_asgi_websocket_receive(PyObject *self, PyObject *none) msg = nxt_py_asgi_new_msg(ws->req, nxt_py_websocket_connect_str); - return nxt_py_asgi_set_result_soon(ws->req, future, msg); + return nxt_py_asgi_set_result_soon(ws->req, ctx_data, future, msg); } if (ws->pending_fins > 0) { msg = nxt_py_asgi_websocket_pop_msg(ws, NULL); - return nxt_py_asgi_set_result_soon(ws->req, future, msg); + return nxt_py_asgi_set_result_soon(ws->req, ctx_data, future, msg); } if (nxt_slow_path(ws->state == NXT_WS_DISCONNECTED)) { msg = nxt_py_asgi_websocket_disconnect_msg(ws); - return nxt_py_asgi_set_result_soon(ws->req, future, msg); + return nxt_py_asgi_set_result_soon(ws->req, ctx_data, future, msg); } ws->receive_future = future; diff --git a/src/python/nxt_python_wsgi.c b/src/python/nxt_python_wsgi.c index 8f692941..da7b183c 100644 --- a/src/python/nxt_python_wsgi.c +++ b/src/python/nxt_python_wsgi.c @@ -38,55 +38,57 @@ */ -typedef struct nxt_python_run_ctx_s nxt_python_run_ctx_t; - typedef struct { PyObject_HEAD -} nxt_py_input_t; + uint64_t content_length; + uint64_t bytes_sent; + PyObject *environ; + PyObject *start_resp; + PyObject *write; + nxt_unit_request_info_t *req; + PyThreadState *thread_state; +} nxt_python_ctx_t; -typedef struct { - PyObject_HEAD -} nxt_py_error_t; + +static int nxt_python_wsgi_ctx_data_alloc(void **pdata); +static void nxt_python_wsgi_ctx_data_free(void *data); +static int nxt_python_wsgi_run(nxt_unit_ctx_t *ctx); +static void nxt_python_wsgi_done(void); static void nxt_python_request_handler(nxt_unit_request_info_t *req); -static PyObject *nxt_python_create_environ(nxt_task_t *task); -static PyObject *nxt_python_get_environ(nxt_python_run_ctx_t *ctx); -static int nxt_python_add_sptr(nxt_python_run_ctx_t *ctx, PyObject *name, +static PyObject *nxt_python_create_environ(nxt_python_app_conf_t *c); +static PyObject *nxt_python_get_environ(nxt_python_ctx_t *pctx); +static int nxt_python_add_sptr(nxt_python_ctx_t *pctx, PyObject *name, nxt_unit_sptr_t *sptr, uint32_t size); -static int nxt_python_add_field(nxt_python_run_ctx_t *ctx, +static int nxt_python_add_field(nxt_python_ctx_t *pctx, nxt_unit_field_t *field, int n, uint32_t vl); static PyObject *nxt_python_field_name(const char *name, uint8_t len); static PyObject *nxt_python_field_value(nxt_unit_field_t *f, int n, uint32_t vl); -static int nxt_python_add_obj(nxt_python_run_ctx_t *ctx, PyObject *name, +static int nxt_python_add_obj(nxt_python_ctx_t *pctx, PyObject *name, PyObject *value); static PyObject *nxt_py_start_resp(PyObject *self, PyObject *args); -static int nxt_python_response_add_field(nxt_python_run_ctx_t *ctx, +static int nxt_python_response_add_field(nxt_python_ctx_t *pctx, PyObject *name, PyObject *value, int i); static int nxt_python_str_buf(PyObject *str, char **buf, uint32_t *len, PyObject **bytes); static PyObject *nxt_py_write(PyObject *self, PyObject *args); -static void nxt_py_input_dealloc(nxt_py_input_t *self); -static PyObject *nxt_py_input_read(nxt_py_input_t *self, PyObject *args); -static PyObject *nxt_py_input_readline(nxt_py_input_t *self, PyObject *args); -static PyObject *nxt_py_input_getline(nxt_python_run_ctx_t *ctx, size_t size); -static PyObject *nxt_py_input_readlines(nxt_py_input_t *self, PyObject *args); +static void nxt_py_input_dealloc(nxt_python_ctx_t *pctx); +static PyObject *nxt_py_input_read(nxt_python_ctx_t *pctx, PyObject *args); +static PyObject *nxt_py_input_readline(nxt_python_ctx_t *pctx, + PyObject *args); +static PyObject *nxt_py_input_getline(nxt_python_ctx_t *pctx, size_t size); +static PyObject *nxt_py_input_readlines(nxt_python_ctx_t *self, + PyObject *args); -static PyObject *nxt_py_input_iter(PyObject *self); -static PyObject *nxt_py_input_next(PyObject *self); +static PyObject *nxt_py_input_iter(PyObject *pctx); +static PyObject *nxt_py_input_next(PyObject *pctx); -static int nxt_python_write(nxt_python_run_ctx_t *ctx, PyObject *bytes); - -struct nxt_python_run_ctx_s { - uint64_t content_length; - uint64_t bytes_sent; - PyObject *environ; - nxt_unit_request_info_t *req; -}; +static int nxt_python_write(nxt_python_ctx_t *pctx, PyObject *bytes); static PyMethodDef nxt_py_start_resp_method[] = { @@ -111,7 +113,7 @@ static PyTypeObject nxt_py_input_type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "unit._input", - .tp_basicsize = sizeof(nxt_py_input_t), + .tp_basicsize = sizeof(nxt_python_ctx_t), .tp_dealloc = (destructor) nxt_py_input_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_doc = "unit input object.", @@ -121,12 +123,7 @@ static PyTypeObject nxt_py_input_type = { }; -static PyObject *nxt_py_start_resp_obj; -static PyObject *nxt_py_write_obj; -static PyObject *nxt_py_environ_ptyp; - -static PyThreadState *nxt_python_thread_state; -static nxt_python_run_ctx_t *nxt_python_run_ctx; +static PyObject *nxt_py_environ_ptyp; static PyObject *nxt_py_80_str; static PyObject *nxt_py_close_str; @@ -143,6 +140,7 @@ static PyObject *nxt_py_server_addr_str; static PyObject *nxt_py_server_name_str; static PyObject *nxt_py_server_port_str; static PyObject *nxt_py_server_protocol_str; +static PyObject *nxt_py_wsgi_input_str; static PyObject *nxt_py_wsgi_uri_scheme_str; static nxt_python_string_t nxt_python_strings[] = { @@ -161,82 +159,132 @@ static nxt_python_string_t nxt_python_strings[] = { { nxt_string("SERVER_NAME"), &nxt_py_server_name_str }, { nxt_string("SERVER_PORT"), &nxt_py_server_port_str }, { nxt_string("SERVER_PROTOCOL"), &nxt_py_server_protocol_str }, + { nxt_string("wsgi.input"), &nxt_py_wsgi_input_str }, { nxt_string("wsgi.url_scheme"), &nxt_py_wsgi_uri_scheme_str }, { nxt_null_string, NULL }, }; +static nxt_python_proto_t nxt_py_wsgi_proto = { + .ctx_data_alloc = nxt_python_wsgi_ctx_data_alloc, + .ctx_data_free = nxt_python_wsgi_ctx_data_free, + .run = nxt_python_wsgi_run, + .done = nxt_python_wsgi_done, +}; + -nxt_int_t -nxt_python_wsgi_init(nxt_task_t *task, nxt_unit_init_t *init) +int +nxt_python_wsgi_init(nxt_unit_init_t *init, nxt_python_proto_t *proto) { PyObject *obj; obj = NULL; - if (nxt_slow_path(nxt_python_init_strings(nxt_python_strings) != NXT_OK)) { - nxt_alert(task, "Python failed to init string objects"); + if (nxt_slow_path(nxt_python_init_strings(nxt_python_strings) + != NXT_UNIT_OK)) + { + nxt_unit_alert(NULL, "Python failed to init string objects"); goto fail; } - obj = PyCFunction_New(nxt_py_start_resp_method, NULL); + obj = nxt_python_create_environ(init->data); if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, - "Python failed to initialize the \"start_response\" function"); goto fail; } - nxt_py_start_resp_obj = obj; + nxt_py_environ_ptyp = obj; + obj = NULL; + + init->callbacks.request_handler = nxt_python_request_handler; + + *proto = nxt_py_wsgi_proto; - obj = PyCFunction_New(nxt_py_write_method, NULL); - if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, "Python failed to initialize the \"write\" function"); - goto fail; + return NXT_UNIT_OK; + +fail: + + Py_XDECREF(obj); + + return NXT_UNIT_ERROR; +} + + +static int +nxt_python_wsgi_ctx_data_alloc(void **pdata) +{ + nxt_python_ctx_t *pctx; + + pctx = PyObject_New(nxt_python_ctx_t, &nxt_py_input_type); + if (nxt_slow_path(pctx == NULL)) { + nxt_unit_alert(NULL, + "Python failed to create the \"wsgi.input\" object"); + return NXT_UNIT_ERROR; } - nxt_py_write_obj = obj; + pctx->write = NULL; - obj = nxt_python_create_environ(task); - if (nxt_slow_path(obj == NULL)) { + pctx->start_resp = PyCFunction_New(nxt_py_start_resp_method, + (PyObject *) pctx); + if (nxt_slow_path(pctx->start_resp == NULL)) { + nxt_unit_alert(NULL, + "Python failed to initialize the \"start_response\" function"); goto fail; } - nxt_py_environ_ptyp = obj; - obj = NULL; + pctx->write = PyCFunction_New(nxt_py_write_method, (PyObject *) pctx); + if (nxt_slow_path(pctx->write == NULL)) { + nxt_unit_alert(NULL, + "Python failed to initialize the \"write\" function"); + goto fail; + } - init->callbacks.request_handler = nxt_python_request_handler; + *pdata = pctx; - return NXT_OK; + return NXT_UNIT_OK; fail: - Py_XDECREF(obj); + nxt_python_wsgi_ctx_data_free(pctx); - return NXT_ERROR; + return NXT_UNIT_ERROR; } -int +static void +nxt_python_wsgi_ctx_data_free(void *data) +{ + nxt_python_ctx_t *pctx; + + pctx = data; + + Py_XDECREF(pctx->start_resp); + Py_XDECREF(pctx->write); + Py_XDECREF(pctx); +} + + +static int nxt_python_wsgi_run(nxt_unit_ctx_t *ctx) { - int rc; + int rc; + nxt_python_ctx_t *pctx; - nxt_python_thread_state = PyEval_SaveThread(); + pctx = ctx->data; + + pctx->thread_state = PyEval_SaveThread(); rc = nxt_unit_run(ctx); - PyEval_RestoreThread(nxt_python_thread_state); + PyEval_RestoreThread(pctx->thread_state); return rc; } -void +static void nxt_python_wsgi_done(void) { nxt_python_done_strings(nxt_python_strings); - Py_XDECREF(nxt_py_start_resp_obj); - Py_XDECREF(nxt_py_write_obj); Py_XDECREF(nxt_py_environ_ptyp); } @@ -244,14 +292,20 @@ nxt_python_wsgi_done(void) static void nxt_python_request_handler(nxt_unit_request_info_t *req) { - int rc; - PyObject *environ, *args, *response, *iterator, *item; - PyObject *close, *result; - nxt_python_run_ctx_t run_ctx = {-1, 0, NULL, req}; + int rc; + PyObject *environ, *args, *response, *iterator, *item; + PyObject *close, *result; + nxt_python_ctx_t *pctx; - PyEval_RestoreThread(nxt_python_thread_state); + pctx = req->ctx->data; - environ = nxt_python_get_environ(&run_ctx); + pctx->content_length = -1; + pctx->bytes_sent = 0; + pctx->req = req; + + PyEval_RestoreThread(pctx->thread_state); + + environ = nxt_python_get_environ(pctx); if (nxt_slow_path(environ == NULL)) { rc = NXT_UNIT_ERROR; goto done; @@ -269,10 +323,8 @@ nxt_python_request_handler(nxt_unit_request_info_t *req) PyTuple_SET_ITEM(args, 0, environ); - Py_INCREF(nxt_py_start_resp_obj); - PyTuple_SET_ITEM(args, 1, nxt_py_start_resp_obj); - - nxt_python_run_ctx = &run_ctx; + Py_INCREF(pctx->start_resp); + PyTuple_SET_ITEM(args, 1, pctx->start_resp); response = PyObject_CallObject(nxt_py_application, args); @@ -288,7 +340,7 @@ nxt_python_request_handler(nxt_unit_request_info_t *req) /* Shortcut: avoid iterate over response string symbols. */ if (PyBytes_Check(response)) { - rc = nxt_python_write(&run_ctx, response); + rc = nxt_python_write(pctx, response); } else { iterator = PyObject_GetIter(response); @@ -296,7 +348,7 @@ nxt_python_request_handler(nxt_unit_request_info_t *req) if (nxt_fast_path(iterator != NULL)) { rc = NXT_UNIT_OK; - while (run_ctx.bytes_sent < run_ctx.content_length) { + while (pctx->bytes_sent < pctx->content_length) { item = PyIter_Next(iterator); if (item == NULL) { @@ -312,7 +364,7 @@ nxt_python_request_handler(nxt_unit_request_info_t *req) } if (nxt_fast_path(PyBytes_Check(item))) { - rc = nxt_python_write(&run_ctx, item); + rc = nxt_python_write(pctx, item); } else { nxt_unit_req_error(req, "the application returned " @@ -361,29 +413,31 @@ nxt_python_request_handler(nxt_unit_request_info_t *req) done: - nxt_python_thread_state = PyEval_SaveThread(); + pctx->thread_state = PyEval_SaveThread(); + + pctx->req = NULL; - nxt_python_run_ctx = NULL; nxt_unit_request_done(req, rc); } static PyObject * -nxt_python_create_environ(nxt_task_t *task) +nxt_python_create_environ(nxt_python_app_conf_t *c) { PyObject *obj, *err, *environ; environ = PyDict_New(); if (nxt_slow_path(environ == NULL)) { - nxt_alert(task, "Python failed to create the \"environ\" dictionary"); + nxt_unit_alert(NULL, + "Python failed to create the \"environ\" dictionary"); return NULL; } obj = PyString_FromStringAndSize((char *) nxt_server.start, nxt_server.length); if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to create the \"SERVER_SOFTWARE\" environ value"); goto fail; } @@ -391,7 +445,7 @@ nxt_python_create_environ(nxt_task_t *task) if (nxt_slow_path(PyDict_SetItemString(environ, "SERVER_SOFTWARE", obj) != 0)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to set the \"SERVER_SOFTWARE\" environ value"); goto fail; } @@ -401,15 +455,15 @@ nxt_python_create_environ(nxt_task_t *task) obj = Py_BuildValue("(ii)", 1, 0); if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to build the \"wsgi.version\" environ value"); goto fail; } if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.version", obj) != 0)) { - nxt_alert(task, - "Python failed to set the \"wsgi.version\" environ value"); + nxt_unit_alert(NULL, + "Python failed to set the \"wsgi.version\" environ value"); goto fail; } @@ -418,10 +472,10 @@ nxt_python_create_environ(nxt_task_t *task) if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.multithread", - Py_False) + c->threads > 1 ? Py_True : Py_False) != 0)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to set the \"wsgi.multithread\" environ value"); goto fail; } @@ -430,7 +484,7 @@ nxt_python_create_environ(nxt_task_t *task) Py_True) != 0)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to set the \"wsgi.multiprocess\" environ value"); goto fail; } @@ -439,46 +493,30 @@ nxt_python_create_environ(nxt_task_t *task) Py_False) != 0)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to set the \"wsgi.run_once\" environ value"); goto fail; } if (nxt_slow_path(PyType_Ready(&nxt_py_input_type) != 0)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to initialize the \"wsgi.input\" type object"); goto fail; } - obj = (PyObject *) PyObject_New(nxt_py_input_t, &nxt_py_input_type); - - if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, "Python failed to create the \"wsgi.input\" object"); - goto fail; - } - - if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.input", obj) != 0)) { - nxt_alert(task, - "Python failed to set the \"wsgi.input\" environ value"); - goto fail; - } - - Py_DECREF(obj); - obj = NULL; - err = PySys_GetObject((char *) "stderr"); if (nxt_slow_path(err == NULL)) { - nxt_alert(task, "Python failed to get \"sys.stderr\" object"); + nxt_unit_alert(NULL, "Python failed to get \"sys.stderr\" object"); goto fail; } if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.errors", err) != 0)) { - nxt_alert(task, - "Python failed to set the \"wsgi.errors\" environ value"); + nxt_unit_alert(NULL, + "Python failed to set the \"wsgi.errors\" environ value"); goto fail; } @@ -494,7 +532,7 @@ fail: static PyObject * -nxt_python_get_environ(nxt_python_run_ctx_t *ctx) +nxt_python_get_environ(nxt_python_ctx_t *pctx) { int rc; uint32_t i, j, vl; @@ -504,15 +542,15 @@ nxt_python_get_environ(nxt_python_run_ctx_t *ctx) environ = PyDict_Copy(nxt_py_environ_ptyp); if (nxt_slow_path(environ == NULL)) { - nxt_unit_req_error(ctx->req, + nxt_unit_req_error(pctx->req, "Python failed to copy the \"environ\" dictionary"); return NULL; } - ctx->environ = environ; + pctx->environ = environ; - r = ctx->req->request; + r = pctx->req->request; #define RC(S) \ do { \ @@ -522,36 +560,36 @@ nxt_python_get_environ(nxt_python_run_ctx_t *ctx) } \ } while(0) - RC(nxt_python_add_sptr(ctx, nxt_py_request_method_str, &r->method, + RC(nxt_python_add_sptr(pctx, nxt_py_request_method_str, &r->method, r->method_length)); - RC(nxt_python_add_sptr(ctx, nxt_py_request_uri_str, &r->target, + RC(nxt_python_add_sptr(pctx, nxt_py_request_uri_str, &r->target, r->target_length)); - RC(nxt_python_add_sptr(ctx, nxt_py_query_string_str, &r->query, + RC(nxt_python_add_sptr(pctx, nxt_py_query_string_str, &r->query, r->query_length)); - RC(nxt_python_add_sptr(ctx, nxt_py_path_info_str, &r->path, + RC(nxt_python_add_sptr(pctx, nxt_py_path_info_str, &r->path, r->path_length)); - RC(nxt_python_add_sptr(ctx, nxt_py_remote_addr_str, &r->remote, + RC(nxt_python_add_sptr(pctx, nxt_py_remote_addr_str, &r->remote, r->remote_length)); - RC(nxt_python_add_sptr(ctx, nxt_py_server_addr_str, &r->local, + RC(nxt_python_add_sptr(pctx, nxt_py_server_addr_str, &r->local, r->local_length)); if (r->tls) { - RC(nxt_python_add_obj(ctx, nxt_py_wsgi_uri_scheme_str, + RC(nxt_python_add_obj(pctx, nxt_py_wsgi_uri_scheme_str, nxt_py_https_str)); } else { - RC(nxt_python_add_obj(ctx, nxt_py_wsgi_uri_scheme_str, + RC(nxt_python_add_obj(pctx, nxt_py_wsgi_uri_scheme_str, nxt_py_http_str)); } - RC(nxt_python_add_sptr(ctx, nxt_py_server_protocol_str, &r->version, + RC(nxt_python_add_sptr(pctx, nxt_py_server_protocol_str, &r->version, r->version_length)); - RC(nxt_python_add_sptr(ctx, nxt_py_server_name_str, &r->server_name, + RC(nxt_python_add_sptr(pctx, nxt_py_server_name_str, &r->server_name, r->server_name_length)); - RC(nxt_python_add_obj(ctx, nxt_py_server_port_str, nxt_py_80_str)); + RC(nxt_python_add_obj(pctx, nxt_py_server_port_str, nxt_py_80_str)); - nxt_unit_request_group_dup_fields(ctx->req); + nxt_unit_request_group_dup_fields(pctx->req); for (i = 0; i < r->fields_count;) { f = r->fields + i; @@ -569,7 +607,7 @@ nxt_python_get_environ(nxt_python_run_ctx_t *ctx) vl += 2 + f2->value_length; } - RC(nxt_python_add_field(ctx, f, j - i, vl)); + RC(nxt_python_add_field(pctx, f, j - i, vl)); i = j; } @@ -577,19 +615,27 @@ nxt_python_get_environ(nxt_python_run_ctx_t *ctx) if (r->content_length_field != NXT_UNIT_NONE_FIELD) { f = r->fields + r->content_length_field; - RC(nxt_python_add_sptr(ctx, nxt_py_content_length_str, &f->value, + RC(nxt_python_add_sptr(pctx, nxt_py_content_length_str, &f->value, f->value_length)); } if (r->content_type_field != NXT_UNIT_NONE_FIELD) { f = r->fields + r->content_type_field; - RC(nxt_python_add_sptr(ctx, nxt_py_content_type_str, &f->value, + RC(nxt_python_add_sptr(pctx, nxt_py_content_type_str, &f->value, f->value_length)); } #undef RC + if (nxt_slow_path(PyDict_SetItem(environ, nxt_py_wsgi_input_str, + (PyObject *) pctx) != 0)) + { + nxt_unit_req_error(pctx->req, + "Python failed to set the \"wsgi.input\" environ value"); + goto fail; + } + return environ; fail: @@ -601,7 +647,7 @@ fail: static int -nxt_python_add_sptr(nxt_python_run_ctx_t *ctx, PyObject *name, +nxt_python_add_sptr(nxt_python_ctx_t *pctx, PyObject *name, nxt_unit_sptr_t *sptr, uint32_t size) { char *src; @@ -611,7 +657,7 @@ nxt_python_add_sptr(nxt_python_run_ctx_t *ctx, PyObject *name, value = PyString_FromStringAndSize(src, size); if (nxt_slow_path(value == NULL)) { - nxt_unit_req_error(ctx->req, + nxt_unit_req_error(pctx->req, "Python failed to create value string \"%.*s\"", (int) size, src); nxt_python_print_exception(); @@ -619,8 +665,8 @@ nxt_python_add_sptr(nxt_python_run_ctx_t *ctx, PyObject *name, return NXT_UNIT_ERROR; } - if (nxt_slow_path(PyDict_SetItem(ctx->environ, name, value) != 0)) { - nxt_unit_req_error(ctx->req, + if (nxt_slow_path(PyDict_SetItem(pctx->environ, name, value) != 0)) { + nxt_unit_req_error(pctx->req, "Python failed to set the \"%s\" environ value", PyUnicode_AsUTF8(name)); Py_DECREF(value); @@ -635,7 +681,7 @@ nxt_python_add_sptr(nxt_python_run_ctx_t *ctx, PyObject *name, static int -nxt_python_add_field(nxt_python_run_ctx_t *ctx, nxt_unit_field_t *field, int n, +nxt_python_add_field(nxt_python_ctx_t *pctx, nxt_unit_field_t *field, int n, uint32_t vl) { char *src; @@ -645,7 +691,7 @@ nxt_python_add_field(nxt_python_run_ctx_t *ctx, nxt_unit_field_t *field, int n, name = nxt_python_field_name(src, field->name_length); if (nxt_slow_path(name == NULL)) { - nxt_unit_req_error(ctx->req, + nxt_unit_req_error(pctx->req, "Python failed to create name string \"%.*s\"", (int) field->name_length, src); nxt_python_print_exception(); @@ -656,7 +702,7 @@ nxt_python_add_field(nxt_python_run_ctx_t *ctx, nxt_unit_field_t *field, int n, value = nxt_python_field_value(field, n, vl); if (nxt_slow_path(value == NULL)) { - nxt_unit_req_error(ctx->req, + nxt_unit_req_error(pctx->req, "Python failed to create value string \"%.*s\"", (int) field->value_length, (char *) nxt_unit_sptr_get(&field->value)); @@ -665,8 +711,8 @@ nxt_python_add_field(nxt_python_run_ctx_t *ctx, nxt_unit_field_t *field, int n, goto fail; } - if (nxt_slow_path(PyDict_SetItem(ctx->environ, name, value) != 0)) { - nxt_unit_req_error(ctx->req, + if (nxt_slow_path(PyDict_SetItem(pctx->environ, name, value) != 0)) { + nxt_unit_req_error(pctx->req, "Python failed to set the \"%s\" environ value", PyUnicode_AsUTF8(name)); goto fail; @@ -761,10 +807,10 @@ nxt_python_field_value(nxt_unit_field_t *f, int n, uint32_t vl) static int -nxt_python_add_obj(nxt_python_run_ctx_t *ctx, PyObject *name, PyObject *value) +nxt_python_add_obj(nxt_python_ctx_t *pctx, PyObject *name, PyObject *value) { - if (nxt_slow_path(PyDict_SetItem(ctx->environ, name, value) != 0)) { - nxt_unit_req_error(ctx->req, + if (nxt_slow_path(PyDict_SetItem(pctx->environ, name, value) != 0)) { + nxt_unit_req_error(pctx->req, "Python failed to set the \"%s\" environ value", PyUnicode_AsUTF8(name)); @@ -778,15 +824,15 @@ nxt_python_add_obj(nxt_python_run_ctx_t *ctx, PyObject *name, PyObject *value) static PyObject * nxt_py_start_resp(PyObject *self, PyObject *args) { - int rc, status; - char *status_str, *space_ptr; - uint32_t status_len; - PyObject *headers, *tuple, *string, *status_bytes; - Py_ssize_t i, n, fields_size, fields_count; - nxt_python_run_ctx_t *ctx; - - ctx = nxt_python_run_ctx; - if (nxt_slow_path(ctx == NULL)) { + int rc, status; + char *status_str, *space_ptr; + uint32_t status_len; + PyObject *headers, *tuple, *string, *status_bytes; + Py_ssize_t i, n, fields_size, fields_count; + nxt_python_ctx_t *pctx; + + pctx = (nxt_python_ctx_t *) self; + if (nxt_slow_path(pctx->req == NULL)) { return PyErr_Format(PyExc_RuntimeError, "start_response() is called " "outside of WSGI request processing"); @@ -851,7 +897,7 @@ nxt_py_start_resp(PyObject *self, PyObject *args) } } - ctx->content_length = -1; + pctx->content_length = -1; string = PyTuple_GET_ITEM(args, 0); rc = nxt_python_str_buf(string, &status_str, &status_len, &status_bytes); @@ -877,7 +923,7 @@ nxt_py_start_resp(PyObject *self, PyObject *args) * ... applications can replace their originally intended output with error * output, up until the last possible moment. */ - rc = nxt_unit_response_init(ctx->req, status, fields_count, fields_size); + rc = nxt_unit_response_init(pctx->req, status, fields_count, fields_size); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return PyErr_Format(PyExc_RuntimeError, "failed to allocate response object"); @@ -886,7 +932,7 @@ nxt_py_start_resp(PyObject *self, PyObject *args) for (i = 0; i < fields_count; i++) { tuple = PyList_GET_ITEM(headers, i); - rc = nxt_python_response_add_field(ctx, PyTuple_GET_ITEM(tuple, 0), + rc = nxt_python_response_add_field(pctx, PyTuple_GET_ITEM(tuple, 0), PyTuple_GET_ITEM(tuple, 1), i); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return PyErr_Format(PyExc_RuntimeError, @@ -907,21 +953,21 @@ nxt_py_start_resp(PyObject *self, PyObject *args) * possible exception to this rule is if the response headers explicitly * include a Content-Length of zero.) */ - if (ctx->content_length == 0) { - rc = nxt_unit_response_send(ctx->req); + if (pctx->content_length == 0) { + rc = nxt_unit_response_send(pctx->req); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return PyErr_Format(PyExc_RuntimeError, "failed to send response headers"); } } - Py_INCREF(nxt_py_write_obj); - return nxt_py_write_obj; + Py_INCREF(pctx->write); + return pctx->write; } static int -nxt_python_response_add_field(nxt_python_run_ctx_t *ctx, PyObject *name, +nxt_python_response_add_field(nxt_python_ctx_t *pctx, PyObject *name, PyObject *value, int i) { int rc; @@ -943,20 +989,20 @@ nxt_python_response_add_field(nxt_python_run_ctx_t *ctx, PyObject *name, goto fail; } - rc = nxt_unit_response_add_field(ctx->req, name_str, name_length, + rc = nxt_unit_response_add_field(pctx->req, name_str, name_length, value_str, value_length); if (nxt_slow_path(rc != NXT_UNIT_OK)) { goto fail; } - if (ctx->req->response->fields[i].hash == NXT_UNIT_HASH_CONTENT_LENGTH) { + if (pctx->req->response->fields[i].hash == NXT_UNIT_HASH_CONTENT_LENGTH) { content_length = nxt_off_t_parse((u_char *) value_str, value_length); if (nxt_slow_path(content_length < 0)) { - nxt_unit_req_error(ctx->req, "failed to parse Content-Length " + nxt_unit_req_error(pctx->req, "failed to parse Content-Length " "value %.*s", (int) value_length, value_str); } else { - ctx->content_length = content_length; + pctx->content_length = content_length; } } @@ -1001,7 +1047,7 @@ nxt_py_write(PyObject *self, PyObject *str) NXT_PYTHON_BYTES_TYPE); } - rc = nxt_python_write(nxt_python_run_ctx, str); + rc = nxt_python_write((nxt_python_ctx_t *) self, str); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return PyErr_Format(PyExc_RuntimeError, "failed to write response value"); @@ -1012,28 +1058,26 @@ nxt_py_write(PyObject *self, PyObject *str) static void -nxt_py_input_dealloc(nxt_py_input_t *self) +nxt_py_input_dealloc(nxt_python_ctx_t *self) { PyObject_Del(self); } static PyObject * -nxt_py_input_read(nxt_py_input_t *self, PyObject *args) +nxt_py_input_read(nxt_python_ctx_t *pctx, PyObject *args) { - char *buf; - PyObject *content, *obj; - Py_ssize_t size, n; - nxt_python_run_ctx_t *ctx; + char *buf; + PyObject *content, *obj; + Py_ssize_t size, n; - ctx = nxt_python_run_ctx; - if (nxt_slow_path(ctx == NULL)) { + if (nxt_slow_path(pctx->req == NULL)) { return PyErr_Format(PyExc_RuntimeError, "wsgi.input.read() is called " "outside of WSGI request processing"); } - size = ctx->req->content_length; + size = pctx->req->content_length; n = PyTuple_GET_SIZE(args); @@ -1057,8 +1101,8 @@ nxt_py_input_read(nxt_py_input_t *self, PyObject *args) } } - if (size == -1 || size > (Py_ssize_t) ctx->req->content_length) { - size = ctx->req->content_length; + if (size == -1 || size > (Py_ssize_t) pctx->req->content_length) { + size = pctx->req->content_length; } } @@ -1069,22 +1113,20 @@ nxt_py_input_read(nxt_py_input_t *self, PyObject *args) buf = PyBytes_AS_STRING(content); - size = nxt_unit_request_read(ctx->req, buf, size); + size = nxt_unit_request_read(pctx->req, buf, size); return content; } static PyObject * -nxt_py_input_readline(nxt_py_input_t *self, PyObject *args) +nxt_py_input_readline(nxt_python_ctx_t *pctx, PyObject *args) { - ssize_t ssize; - PyObject *obj; - Py_ssize_t n; - nxt_python_run_ctx_t *ctx; + ssize_t ssize; + PyObject *obj; + Py_ssize_t n; - ctx = nxt_python_run_ctx; - if (nxt_slow_path(ctx == NULL)) { + if (nxt_slow_path(pctx->req == NULL)) { return PyErr_Format(PyExc_RuntimeError, "wsgi.input.readline() is called " "outside of WSGI request processing"); @@ -1102,7 +1144,7 @@ nxt_py_input_readline(nxt_py_input_t *self, PyObject *args) ssize = PyNumber_AsSsize_t(obj, PyExc_OverflowError); if (nxt_fast_path(ssize > 0)) { - return nxt_py_input_getline(ctx, ssize); + return nxt_py_input_getline(pctx, ssize); } if (ssize == 0) { @@ -1119,18 +1161,18 @@ nxt_py_input_readline(nxt_py_input_t *self, PyObject *args) } } - return nxt_py_input_getline(ctx, SSIZE_MAX); + return nxt_py_input_getline(pctx, SSIZE_MAX); } static PyObject * -nxt_py_input_getline(nxt_python_run_ctx_t *ctx, size_t size) +nxt_py_input_getline(nxt_python_ctx_t *pctx, size_t size) { void *buf; ssize_t res; PyObject *content; - res = nxt_unit_request_readline_size(ctx->req, size); + res = nxt_unit_request_readline_size(pctx->req, size); if (nxt_slow_path(res < 0)) { return NULL; } @@ -1146,20 +1188,18 @@ nxt_py_input_getline(nxt_python_run_ctx_t *ctx, size_t size) buf = PyBytes_AS_STRING(content); - res = nxt_unit_request_read(ctx->req, buf, res); + res = nxt_unit_request_read(pctx->req, buf, res); return content; } static PyObject * -nxt_py_input_readlines(nxt_py_input_t *self, PyObject *args) +nxt_py_input_readlines(nxt_python_ctx_t *pctx, PyObject *args) { - PyObject *res; - nxt_python_run_ctx_t *ctx; + PyObject *res; - ctx = nxt_python_run_ctx; - if (nxt_slow_path(ctx == NULL)) { + if (nxt_slow_path(pctx->req == NULL)) { return PyErr_Format(PyExc_RuntimeError, "wsgi.input.readlines() is called " "outside of WSGI request processing"); @@ -1171,7 +1211,7 @@ nxt_py_input_readlines(nxt_py_input_t *self, PyObject *args) } for ( ;; ) { - PyObject *line = nxt_py_input_getline(ctx, SSIZE_MAX); + PyObject *line = nxt_py_input_getline(pctx, SSIZE_MAX); if (nxt_slow_path(line == NULL)) { Py_DECREF(res); return NULL; @@ -1201,17 +1241,17 @@ nxt_py_input_iter(PyObject *self) static PyObject * nxt_py_input_next(PyObject *self) { - PyObject *line; - nxt_python_run_ctx_t *ctx; + PyObject *line; + nxt_python_ctx_t *pctx; - ctx = nxt_python_run_ctx; - if (nxt_slow_path(ctx == NULL)) { + pctx = (nxt_python_ctx_t *) self; + if (nxt_slow_path(pctx->req == NULL)) { return PyErr_Format(PyExc_RuntimeError, "wsgi.input.next() is called " "outside of WSGI request processing"); } - line = nxt_py_input_getline(ctx, SSIZE_MAX); + line = nxt_py_input_getline(pctx, SSIZE_MAX); if (nxt_slow_path(line == NULL)) { return NULL; } @@ -1227,7 +1267,7 @@ nxt_py_input_next(PyObject *self) static int -nxt_python_write(nxt_python_run_ctx_t *ctx, PyObject *bytes) +nxt_python_write(nxt_python_ctx_t *pctx, PyObject *bytes) { int rc; char *str_buf; @@ -1248,16 +1288,16 @@ nxt_python_write(nxt_python_run_ctx_t *ctx, PyObject *bytes) * stop iterating over the response when enough data has been sent, or raise * an error if the application tries to write() past that point. */ - if (nxt_slow_path(str_length > ctx->content_length - ctx->bytes_sent)) { - nxt_unit_req_error(ctx->req, "content length %"PRIu64" exceeded", - ctx->content_length); + if (nxt_slow_path(str_length > pctx->content_length - pctx->bytes_sent)) { + nxt_unit_req_error(pctx->req, "content length %"PRIu64" exceeded", + pctx->content_length); return NXT_UNIT_ERROR; } - rc = nxt_unit_response_write(ctx->req, str_buf, str_length); + rc = nxt_unit_response_write(pctx->req, str_buf, str_length); if (nxt_fast_path(rc == NXT_UNIT_OK)) { - ctx->bytes_sent += str_length; + pctx->bytes_sent += str_length; } return rc; -- cgit From 8e37b1cbf5ad2935c1777992ebded183e6454ba4 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 5 Nov 2020 00:05:00 +0300 Subject: Python: fixing some arguments reference counting. --- src/python/nxt_python_asgi.c | 163 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 130 insertions(+), 33 deletions(-) diff --git a/src/python/nxt_python_asgi.c b/src/python/nxt_python_asgi.c index a188ff56..cab73239 100644 --- a/src/python/nxt_python_asgi.c +++ b/src/python/nxt_python_asgi.c @@ -330,7 +330,7 @@ nxt_python_asgi_run(nxt_unit_ctx_t *ctx) static void nxt_py_asgi_remove_reader(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) { - PyObject *res; + PyObject *res, *fd; nxt_py_asgi_ctx_data_t *ctx_data; if (port == NULL || port->in_fd == -1) { @@ -341,16 +341,24 @@ nxt_py_asgi_remove_reader(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) nxt_unit_debug(ctx, "asgi_remove_reader %d %p", port->in_fd, port); - res = PyObject_CallFunctionObjArgs(ctx_data->loop_remove_reader, - PyLong_FromLong(port->in_fd), NULL); + fd = PyLong_FromLong(port->in_fd); + if (nxt_slow_path(fd == NULL)) { + nxt_unit_alert(ctx, "Python failed to create Long object"); + nxt_python_print_exception(); + + return; + } + + res = PyObject_CallFunctionObjArgs(ctx_data->loop_remove_reader, fd, NULL); if (nxt_slow_path(res == NULL)) { nxt_unit_alert(ctx, "Python failed to remove_reader"); nxt_python_print_exception(); - return; + } else { + Py_DECREF(res); } - Py_DECREF(res); + Py_DECREF(fd); } @@ -789,7 +797,8 @@ fail: static int nxt_python_asgi_ready(nxt_unit_ctx_t *ctx) { - PyObject *res; + int rc; + PyObject *res, *fd, *py_ctx, *py_port; nxt_unit_port_t *port; nxt_py_asgi_ctx_data_t *ctx_data; @@ -803,29 +812,64 @@ nxt_python_asgi_ready(nxt_unit_ctx_t *ctx) ctx_data = ctx->data; + rc = NXT_UNIT_ERROR; + + fd = PyLong_FromLong(port->in_fd); + if (nxt_slow_path(fd == NULL)) { + nxt_unit_alert(ctx, "Python failed to create fd"); + nxt_python_print_exception(); + + return rc; + } + + py_ctx = PyLong_FromVoidPtr(ctx); + if (nxt_slow_path(py_ctx == NULL)) { + nxt_unit_alert(ctx, "Python failed to create py_ctx"); + nxt_python_print_exception(); + + goto clean_fd; + } + + py_port = PyLong_FromVoidPtr(port); + if (nxt_slow_path(py_port == NULL)) { + nxt_unit_alert(ctx, "Python failed to create py_port"); + nxt_python_print_exception(); + + goto clean_py_ctx; + } + res = PyObject_CallFunctionObjArgs(ctx_data->loop_add_reader, - PyLong_FromLong(port->in_fd), - nxt_py_port_read, - PyLong_FromVoidPtr(ctx), - PyLong_FromVoidPtr(port), NULL); + fd, nxt_py_port_read, + py_ctx, py_port, NULL); if (nxt_slow_path(res == NULL)) { nxt_unit_alert(ctx, "Python failed to add_reader"); nxt_python_print_exception(); - return NXT_UNIT_ERROR; + } else { + Py_DECREF(res); + + rc = NXT_UNIT_OK; } - Py_DECREF(res); + Py_DECREF(py_port); - return NXT_UNIT_OK; +clean_py_ctx: + + Py_DECREF(py_ctx); + +clean_fd: + + Py_DECREF(fd); + + return rc; } static int nxt_py_asgi_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) { - int nb; - PyObject *res; + int nb, rc; + PyObject *res, *fd, *py_ctx, *py_port; nxt_py_asgi_ctx_data_t *ctx_data; if (port->in_fd == -1) { @@ -854,21 +898,56 @@ nxt_py_asgi_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) ctx_data->port = port; port->data = ctx_data; + rc = NXT_UNIT_ERROR; + + fd = PyLong_FromLong(port->in_fd); + if (nxt_slow_path(fd == NULL)) { + nxt_unit_alert(ctx, "Python failed to create fd"); + nxt_python_print_exception(); + + return rc; + } + + py_ctx = PyLong_FromVoidPtr(ctx); + if (nxt_slow_path(py_ctx == NULL)) { + nxt_unit_alert(ctx, "Python failed to create py_ctx"); + nxt_python_print_exception(); + + goto clean_fd; + } + + py_port = PyLong_FromVoidPtr(port); + if (nxt_slow_path(py_port == NULL)) { + nxt_unit_alert(ctx, "Python failed to create py_port"); + nxt_python_print_exception(); + + goto clean_py_ctx; + } + res = PyObject_CallFunctionObjArgs(ctx_data->loop_add_reader, - PyLong_FromLong(port->in_fd), - nxt_py_port_read, - PyLong_FromVoidPtr(ctx), - PyLong_FromVoidPtr(port), NULL); + fd, nxt_py_port_read, + py_ctx, py_port, NULL); if (nxt_slow_path(res == NULL)) { nxt_unit_alert(ctx, "Python failed to add_reader"); nxt_python_print_exception(); - return NXT_UNIT_ERROR; + } else { + Py_DECREF(res); + + rc = NXT_UNIT_OK; } - Py_DECREF(res); + Py_DECREF(py_port); - return NXT_UNIT_OK; +clean_py_ctx: + + Py_DECREF(py_ctx); + +clean_fd: + + Py_DECREF(fd); + + return rc; } @@ -890,7 +969,7 @@ nxt_py_asgi_remove_port(nxt_unit_t *lib, nxt_unit_port_t *port) static void nxt_py_asgi_quit(nxt_unit_ctx_t *ctx) { - PyObject *res; + PyObject *res, *p; nxt_py_asgi_ctx_data_t *ctx_data; nxt_unit_debug(ctx, "asgi_quit %p", ctx); @@ -898,25 +977,43 @@ nxt_py_asgi_quit(nxt_unit_ctx_t *ctx) ctx_data = ctx->data; if (nxt_py_shared_port != NULL) { - res = PyObject_CallFunctionObjArgs(ctx_data->loop_remove_reader, - PyLong_FromLong(nxt_py_shared_port->in_fd), NULL); - if (nxt_slow_path(res == NULL)) { - nxt_unit_alert(NULL, "Python failed to remove_reader"); + p = PyLong_FromLong(nxt_py_shared_port->in_fd); + if (nxt_slow_path(p == NULL)) { + nxt_unit_alert(NULL, "Python failed to create Long"); nxt_python_print_exception(); } else { - Py_DECREF(res); + res = PyObject_CallFunctionObjArgs(ctx_data->loop_remove_reader, + p, NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(NULL, "Python failed to remove_reader"); + nxt_python_print_exception(); + + } else { + Py_DECREF(res); + } + + Py_DECREF(p); } } - res = PyObject_CallFunctionObjArgs(ctx_data->quit_future_set_result, - PyLong_FromLong(0), NULL); - if (nxt_slow_path(res == NULL)) { - nxt_unit_alert(ctx, "Python failed to set_result"); + p = PyLong_FromLong(0); + if (nxt_slow_path(p == NULL)) { + nxt_unit_alert(NULL, "Python failed to create Long"); nxt_python_print_exception(); } else { - Py_DECREF(res); + res = PyObject_CallFunctionObjArgs(ctx_data->quit_future_set_result, + p, NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(ctx, "Python failed to set_result"); + nxt_python_print_exception(); + + } else { + Py_DECREF(res); + } + + Py_DECREF(p); } } -- cgit From f27953af6103db390aa3eee012a254f130fdf5f4 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 5 Nov 2020 00:05:02 +0300 Subject: Tests: added Python threading tests. --- test/python/threads/asgi.py | 27 +++++++++++++++++++++++++++ test/python/threads/wsgi.py | 15 +++++++++++++++ test/test_asgi_application.py | 41 +++++++++++++++++++++++++++++++++++++++++ test/test_python_application.py | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+) create mode 100644 test/python/threads/asgi.py create mode 100644 test/python/threads/wsgi.py diff --git a/test/python/threads/asgi.py b/test/python/threads/asgi.py new file mode 100644 index 00000000..d51ae431 --- /dev/null +++ b/test/python/threads/asgi.py @@ -0,0 +1,27 @@ +import asyncio +import time +import threading + +async def application(scope, receive, send): + assert scope['type'] == 'http' + + headers = scope.get('headers', []) + + def get_header(n, v=None): + for h in headers: + if h[0] == n: + return h[1] + return v + + delay = float(get_header(b'x-delay', 0)) + + time.sleep(delay) + + await send({ + 'type': 'http.response.start', + 'status': 200, + 'headers': [ + (b'content-length', b'0'), + (b'x-thread', str(threading.currentThread().ident).encode()), + ] + }) diff --git a/test/python/threads/wsgi.py b/test/python/threads/wsgi.py new file mode 100644 index 00000000..1cc8ffe2 --- /dev/null +++ b/test/python/threads/wsgi.py @@ -0,0 +1,15 @@ +import time +import threading + +def application(environ, start_response): + delay = float(environ.get('HTTP_X_DELAY', 0)) + + time.sleep(delay) + + start_response('200', [ + ('Content-Length', '0'), + ('Wsgi-Multithread', str(environ['wsgi.multithread'])), + ('X-Thread', str(threading.currentThread().ident)) + ]) + + return [] diff --git a/test/test_asgi_application.py b/test/test_asgi_application.py index 1a654ebf..179a69ee 100644 --- a/test/test_asgi_application.py +++ b/test/test_asgi_application.py @@ -401,3 +401,44 @@ Connection: close assert ( self.wait_for_record(r'\(5\) Thread: 100') is not None ), 'last thread finished' + + def test_asgi_application_threads(self): + self.load('threads') + + assert 'success' in self.conf( + '4', 'applications/threads/threads' + ), 'configure 4 threads' + + socks = [] + + for i in range(4): + (_, sock) = self.get( + headers={ + 'Host': 'localhost', + 'X-Delay': '2', + 'Connection': 'close', + }, + no_recv=True, + start=True, + ) + + socks.append(sock) + + time.sleep(0.25) # required to avoid greedy request reading + + threads = set() + + for sock in socks: + resp = self.recvall(sock).decode('utf-8') + + self.log_in(resp) + + resp = self._resp_to_dict(resp) + + assert resp['status'] == 200, 'status' + + threads.add(resp['headers']['x-thread']) + + sock.close() + + assert len(socks) == len(threads), 'threads differs' diff --git a/test/test_python_application.py b/test/test_python_application.py index 6ea9f4ff..780b836a 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -832,3 +832,44 @@ last line: 987654321 assert 'success' in self.conf_delete('applications/callable/callable') assert self.get()['status'] == 204, 'default response 2' + + def test_python_application_threads(self): + self.load('threads') + + assert 'success' in self.conf( + '4', 'applications/threads/threads' + ), 'configure 4 threads' + + socks = [] + + for i in range(4): + (_, sock) = self.get( + headers={ + 'Host': 'localhost', + 'X-Delay': '2', + 'Connection': 'close', + }, + no_recv=True, + start=True, + ) + + socks.append(sock) + + threads = set() + + for sock in socks: + resp = self.recvall(sock).decode('utf-8') + + self.log_in(resp) + + resp = self._resp_to_dict(resp) + + assert resp['status'] == 200, 'status' + + threads.add(resp['headers']['X-Thread']) + + assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread' + + sock.close() + + assert len(socks) == len(threads), 'threads differs' -- cgit From 29db46c52ba0f05706d83ed75d88e4b57bac36e5 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 5 Nov 2020 00:06:10 +0300 Subject: Java: request processing in multiple threads. This closes #458 issue on GitHub. --- src/nxt_application.h | 2 + src/nxt_conf_validation.c | 8 ++ src/nxt_java.c | 199 ++++++++++++++++++++++++++++++++++++++++++---- src/nxt_main_process.c | 10 +++ 4 files changed, 205 insertions(+), 14 deletions(-) diff --git a/src/nxt_application.h b/src/nxt_application.h index 7f8ec1ba..08de0fce 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -77,6 +77,8 @@ typedef struct { char *webapp; nxt_conf_value_t *options; char *unit_jars; + uint32_t threads; + uint32_t thread_stack_size; } nxt_java_app_conf_t; diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 97a13e38..1ce2d9ea 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -639,6 +639,14 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_java_members[] = { }, { .name = nxt_string("unit_jars"), .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("threads"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_threads, + }, { + .name = nxt_string("thread_stack_size"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_thread_stack_size, }, NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) diff --git a/src/nxt_java.c b/src/nxt_java.c index 1f8864bd..ac715c0b 100644 --- a/src/nxt_java.c +++ b/src/nxt_java.c @@ -36,6 +36,11 @@ static nxt_int_t nxt_java_start(nxt_task_t *task, static void nxt_java_request_handler(nxt_unit_request_info_t *req); static void nxt_java_websocket_handler(nxt_unit_websocket_frame_t *ws); static void nxt_java_close_handler(nxt_unit_request_info_t *req); +static int nxt_java_ready_handler(nxt_unit_ctx_t *ctx); +static void *nxt_java_thread_func(void *main_ctx); +static int nxt_java_init_threads(nxt_java_app_conf_t *c); +static void nxt_java_join_threads(nxt_unit_ctx_t *ctx, + nxt_java_app_conf_t *c); static uint32_t compat[] = { NXT_VERNUM, NXT_DEBUG, @@ -43,6 +48,9 @@ static uint32_t compat[] = { char *nxt_java_modules; +static pthread_t *nxt_java_threads; +static pthread_attr_t *nxt_java_thread_attr; + #define NXT_STRING(x) _NXT_STRING(x) #define _NXT_STRING(x) #x @@ -59,8 +67,10 @@ NXT_EXPORT nxt_app_module_t nxt_app_module = { }; typedef struct { - JNIEnv *env; - jobject ctx; + JavaVM *jvm; + jobject cl; + jobject ctx; + nxt_java_app_conf_t *conf; } nxt_java_data_t; @@ -402,8 +412,10 @@ nxt_java_start(nxt_task_t *task, nxt_process_data_t *data) goto env_failed; } - java_data.env = env; + java_data.jvm = jvm; + java_data.cl = cl; java_data.ctx = nxt_java_startContext(env, c->webapp, classpath); + java_data.conf = c; if ((*env)->ExceptionCheck(env)) { nxt_alert(task, "Unhandled exception in application start"); @@ -411,13 +423,20 @@ nxt_java_start(nxt_task_t *task, nxt_process_data_t *data) return NXT_ERROR; } + rc = nxt_java_init_threads(c); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + return NXT_ERROR; + } + nxt_unit_default_init(task, &java_init); java_init.callbacks.request_handler = nxt_java_request_handler; java_init.callbacks.websocket_handler = nxt_java_websocket_handler; java_init.callbacks.close_handler = nxt_java_close_handler; + java_init.callbacks.ready_handler = nxt_java_ready_handler; java_init.request_data_size = sizeof(nxt_java_request_data_t); java_init.data = &java_data; + java_init.ctx_data = env; java_init.shm_limit = app_conf->shm_limit; ctx = nxt_unit_init(&java_init); @@ -427,9 +446,8 @@ nxt_java_start(nxt_task_t *task, nxt_process_data_t *data) } rc = nxt_unit_run(ctx); - if (nxt_slow_path(rc != NXT_UNIT_OK)) { - /* TODO report error */ - } + + nxt_java_join_threads(ctx, c); nxt_java_stopContext(env, java_data.ctx); @@ -441,7 +459,7 @@ nxt_java_start(nxt_task_t *task, nxt_process_data_t *data) (*jvm)->DestroyJavaVM(jvm); - exit(0); + exit(rc); return NXT_OK; @@ -464,7 +482,7 @@ nxt_java_request_handler(nxt_unit_request_info_t *req) nxt_java_request_data_t *data; java_data = req->unit->data; - env = java_data->env; + env = req->ctx->data; data = req->data; jreq = nxt_java_newRequest(env, java_data->ctx, req); @@ -543,11 +561,9 @@ nxt_java_websocket_handler(nxt_unit_websocket_frame_t *ws) void *b; JNIEnv *env; jobject jbuf; - nxt_java_data_t *java_data; nxt_java_request_data_t *data; - java_data = ws->req->unit->data; - env = java_data->env; + env = ws->req->ctx->data; data = ws->req->data; b = malloc(ws->payload_len); @@ -578,11 +594,9 @@ static void nxt_java_close_handler(nxt_unit_request_info_t *req) { JNIEnv *env; - nxt_java_data_t *java_data; nxt_java_request_data_t *data; - java_data = req->unit->data; - env = java_data->env; + env = req->ctx->data; data = req->data; nxt_java_Request_close(env, data->jreq); @@ -593,3 +607,160 @@ nxt_java_close_handler(nxt_unit_request_info_t *req) nxt_unit_request_done(req, NXT_UNIT_OK); } + +static int +nxt_java_ready_handler(nxt_unit_ctx_t *ctx) +{ + int res; + uint32_t i; + nxt_java_data_t *java_data; + nxt_java_app_conf_t *c; + + /* Worker thread context. */ + if (!nxt_unit_is_main_ctx(ctx)) { + return NXT_UNIT_OK; + } + + java_data = ctx->unit->data; + c = java_data->conf; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + for (i = 0; i < c->threads - 1; i++) { + res = pthread_create(&nxt_java_threads[i], nxt_java_thread_attr, + nxt_java_thread_func, ctx); + + if (nxt_fast_path(res == 0)) { + nxt_unit_debug(ctx, "thread #%d created", (int) (i + 1)); + + } else { + nxt_unit_alert(ctx, "thread #%d create failed: %s (%d)", + (int) (i + 1), strerror(res), res); + + return NXT_UNIT_ERROR; + } + } + + return NXT_UNIT_OK; +} + + +static void * +nxt_java_thread_func(void *data) +{ + int rc; + JavaVM *jvm; + JNIEnv *env; + nxt_unit_ctx_t *main_ctx, *ctx; + nxt_java_data_t *java_data; + + main_ctx = data; + + nxt_unit_debug(main_ctx, "worker thread start"); + + java_data = main_ctx->unit->data; + jvm = java_data->jvm; + + rc = (*jvm)->AttachCurrentThread(jvm, (void **) &env, NULL); + if (rc != JNI_OK) { + nxt_unit_alert(main_ctx, "failed to attach Java VM: %d", (int) rc); + return NULL; + } + + nxt_java_setContextClassLoader(env, java_data->cl); + + ctx = nxt_unit_ctx_alloc(main_ctx, env); + if (nxt_slow_path(ctx == NULL)) { + goto fail; + } + + (void) nxt_unit_run(ctx); + + nxt_unit_done(ctx); + +fail: + + (*jvm)->DetachCurrentThread(jvm); + + nxt_unit_debug(NULL, "worker thread end"); + + return NULL; +} + + +static int +nxt_java_init_threads(nxt_java_app_conf_t *c) +{ + int res; + static pthread_attr_t attr; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + if (c->thread_stack_size > 0) { + res = pthread_attr_init(&attr); + if (nxt_slow_path(res != 0)) { + nxt_unit_alert(NULL, "thread attr init failed: %s (%d)", + strerror(res), res); + + return NXT_UNIT_ERROR; + } + + res = pthread_attr_setstacksize(&attr, c->thread_stack_size); + if (nxt_slow_path(res != 0)) { + nxt_unit_alert(NULL, "thread attr set stack size failed: %s (%d)", + strerror(res), res); + + return NXT_UNIT_ERROR; + } + + nxt_java_thread_attr = &attr; + } + + nxt_java_threads = nxt_unit_malloc(NULL, + sizeof(pthread_t) * (c->threads - 1)); + if (nxt_slow_path(nxt_java_threads == NULL)) { + nxt_unit_alert(NULL, "Failed to allocate thread id array"); + + return NXT_UNIT_ERROR; + } + + memset(nxt_java_threads, 0, sizeof(pthread_t) * (c->threads - 1)); + + return NXT_UNIT_OK; +} + + +static void +nxt_java_join_threads(nxt_unit_ctx_t *ctx, nxt_java_app_conf_t *c) +{ + int res; + uint32_t i; + + if (nxt_java_threads == NULL) { + return; + } + + for (i = 0; i < c->threads - 1; i++) { + if ((uintptr_t) nxt_java_threads[i] == 0) { + continue; + } + + res = pthread_join(nxt_java_threads[i], NULL); + + if (nxt_fast_path(res == 0)) { + nxt_unit_debug(ctx, "thread #%d joined", (int) i); + + } else { + nxt_unit_alert(ctx, "thread #%d join failed: %s (%d)", + (int) i, strerror(res), res); + } + } + + nxt_unit_free(ctx, nxt_java_threads); +} + + diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 16c405ac..ce8d6916 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -266,6 +266,16 @@ static nxt_conf_map_t nxt_java_app_conf[] = { NXT_CONF_MAP_CSTRZ, offsetof(nxt_common_app_conf_t, u.java.unit_jars), }, + { + nxt_string("threads"), + NXT_CONF_MAP_INT32, + offsetof(nxt_common_app_conf_t, u.java.threads), + }, + { + nxt_string("thread_stack_size"), + NXT_CONF_MAP_INT32, + offsetof(nxt_common_app_conf_t, u.java.thread_stack_size), + }, }; -- cgit From 28f1eb55e7c5d60ab4f705eb27e67d31f3fc4ad0 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 5 Nov 2020 00:06:13 +0300 Subject: Tests: added Java threading tests. --- test/java/threads/app.java | 32 ++++++++++++++++++++++++++++++++ test/test_java_application.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 test/java/threads/app.java diff --git a/test/java/threads/app.java b/test/java/threads/app.java new file mode 100644 index 00000000..d0dd3fcc --- /dev/null +++ b/test/java/threads/app.java @@ -0,0 +1,32 @@ + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@WebServlet("/") +public class app extends HttpServlet +{ + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws IOException, ServletException + { + int delay = 0; + + String x_delay = request.getHeader("X-Delay"); + if (x_delay != null) { + delay = Integer.parseInt(x_delay); + } + + try { + Thread.sleep(delay * 1000); + } catch (InterruptedException ex) { + ex.printStackTrace(); + } + + response.addHeader("X-Thread", "" + Thread.currentThread().getId()); + } +} diff --git a/test/test_java_application.py b/test/test_java_application.py index efafa6a1..41345e87 100644 --- a/test/test_java_application.py +++ b/test/test_java_application.py @@ -1012,3 +1012,44 @@ class TestJavaApplication(TestApplicationJava): ) is not None ), 'file created' + + def test_java_application_threads(self): + self.load('threads') + + assert 'success' in self.conf( + '4', 'applications/threads/threads' + ), 'configure 4 threads' + + socks = [] + + for i in range(4): + (_, sock) = self.get( + headers={ + 'Host': 'localhost', + 'X-Delay': '2', + 'Connection': 'close', + }, + no_recv=True, + start=True, + ) + + socks.append(sock) + + time.sleep(0.25) # required to avoid greedy request reading + + threads = set() + + for sock in socks: + resp = self.recvall(sock).decode('utf-8') + + self.log_in(resp) + + resp = self._resp_to_dict(resp) + + assert resp['status'] == 200, 'status' + + threads.add(resp['headers']['X-Thread']) + + sock.close() + + assert len(socks) == len(threads), 'threads differs' -- cgit From 9f8b746e776031e6eef6dea84c8dafbb8c24c725 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 5 Nov 2020 12:45:08 +0300 Subject: Ruby: reusing static constant references to string objects. This shall save a couple of CPU cycles in request processing. --- src/ruby/nxt_ruby.c | 158 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 115 insertions(+), 43 deletions(-) diff --git a/src/ruby/nxt_ruby.c b/src/ruby/nxt_ruby.c index 743bf646..7b383557 100644 --- a/src/ruby/nxt_ruby.c +++ b/src/ruby/nxt_ruby.c @@ -14,14 +14,6 @@ #define NXT_RUBY_RACK_API_VERSION_MAJOR 1 #define NXT_RUBY_RACK_API_VERSION_MINOR 3 -#define NXT_RUBY_STRINGIZE_HELPER(x) #x -#define NXT_RUBY_STRINGIZE(x) NXT_RUBY_STRINGIZE_HELPER(x) - -#define NXT_RUBY_LIB_VERSION \ - NXT_RUBY_STRINGIZE(RUBY_API_VERSION_MAJOR) \ - "." NXT_RUBY_STRINGIZE(RUBY_API_VERSION_MINOR) \ - "." NXT_RUBY_STRINGIZE(RUBY_API_VERSION_TEENY) - typedef struct { nxt_task_t *task; @@ -45,10 +37,8 @@ static void nxt_ruby_request_handler(nxt_unit_request_info_t *req); static VALUE nxt_ruby_rack_app_run(VALUE arg); static int nxt_ruby_read_request(VALUE hash_env); -nxt_inline void nxt_ruby_add_sptr(VALUE hash_env, - const char *name, uint32_t name_len, nxt_unit_sptr_t *sptr, uint32_t len); -nxt_inline void nxt_ruby_add_str(VALUE hash_env, - const char *name, uint32_t name_len, const char *str, uint32_t len); +nxt_inline void nxt_ruby_add_sptr(VALUE hash_env, VALUE name, + nxt_unit_sptr_t *sptr, uint32_t len); static nxt_int_t nxt_ruby_rack_result_status(VALUE result); static int nxt_ruby_rack_result_headers(VALUE result, nxt_int_t status); static int nxt_ruby_hash_info(VALUE r_key, VALUE r_value, VALUE arg); @@ -86,6 +76,93 @@ NXT_EXPORT nxt_app_module_t nxt_app_module = { nxt_ruby_start, }; +typedef struct { + nxt_str_t string; + VALUE *v; +} nxt_ruby_string_t; + +static VALUE nxt_rb_80_str; +static VALUE nxt_rb_content_length_str; +static VALUE nxt_rb_content_type_str; +static VALUE nxt_rb_http_str; +static VALUE nxt_rb_https_str; +static VALUE nxt_rb_path_info_str; +static VALUE nxt_rb_query_string_str; +static VALUE nxt_rb_rack_url_scheme_str; +static VALUE nxt_rb_remote_addr_str; +static VALUE nxt_rb_request_method_str; +static VALUE nxt_rb_request_uri_str; +static VALUE nxt_rb_server_addr_str; +static VALUE nxt_rb_server_name_str; +static VALUE nxt_rb_server_port_str; +static VALUE nxt_rb_server_protocol_str; + +static nxt_ruby_string_t nxt_rb_strings[] = { + { nxt_string("80"), &nxt_rb_80_str }, + { nxt_string("CONTENT_LENGTH"), &nxt_rb_content_length_str }, + { nxt_string("CONTENT_TYPE"), &nxt_rb_content_type_str }, + { nxt_string("http"), &nxt_rb_http_str }, + { nxt_string("https"), &nxt_rb_https_str }, + { nxt_string("PATH_INFO"), &nxt_rb_path_info_str }, + { nxt_string("QUERY_STRING"), &nxt_rb_query_string_str }, + { nxt_string("rack.url_scheme"), &nxt_rb_rack_url_scheme_str }, + { nxt_string("REMOTE_ADDR"), &nxt_rb_remote_addr_str }, + { nxt_string("REQUEST_METHOD"), &nxt_rb_request_method_str }, + { nxt_string("REQUEST_URI"), &nxt_rb_request_uri_str }, + { nxt_string("SERVER_ADDR"), &nxt_rb_server_addr_str }, + { nxt_string("SERVER_NAME"), &nxt_rb_server_name_str }, + { nxt_string("SERVER_PORT"), &nxt_rb_server_port_str }, + { nxt_string("SERVER_PROTOCOL"), &nxt_rb_server_protocol_str }, + { nxt_null_string, NULL }, +}; + + +static int +nxt_ruby_init_strings(void) +{ + VALUE v; + nxt_ruby_string_t *pstr; + + pstr = nxt_rb_strings; + + while (pstr->string.start != NULL) { + v = rb_str_new_static((char *) pstr->string.start, pstr->string.length); + + if (nxt_slow_path(v == Qnil)) { + nxt_unit_alert(NULL, "Ruby: failed to create const string '%.*s'", + (int) pstr->string.length, + (char *) pstr->string.start); + + return NXT_UNIT_ERROR; + } + + *pstr->v = v; + + rb_gc_register_address(pstr->v); + + pstr++; + } + + return NXT_UNIT_OK; +} + + +static void +nxt_ruby_done_strings(void) +{ + nxt_ruby_string_t *pstr; + + pstr = nxt_rb_strings; + + while (pstr->string.start != NULL) { + rb_gc_unregister_address(pstr->v); + + *pstr->v = Qnil; + + pstr++; + } +} + static nxt_int_t nxt_ruby_start(nxt_task_t *task, nxt_process_data_t *data) @@ -109,6 +186,8 @@ nxt_ruby_start(nxt_task_t *task, nxt_process_data_t *data) rack_init.task = task; rack_init.script = &conf->u.ruby.script; + nxt_ruby_init_strings(); + res = rb_protect(nxt_ruby_init_basic, (VALUE) (uintptr_t) &rack_init, &state); if (nxt_slow_path(res == Qnil || state != 0)) { @@ -440,78 +519,69 @@ fail: static int nxt_ruby_read_request(VALUE hash_env) { + VALUE name; uint32_t i; nxt_unit_field_t *f; nxt_unit_request_t *r; r = nxt_ruby_run_ctx.req->request; -#define NL(S) (S), sizeof(S)-1 - - nxt_ruby_add_sptr(hash_env, NL("REQUEST_METHOD"), &r->method, + nxt_ruby_add_sptr(hash_env, nxt_rb_request_method_str, &r->method, r->method_length); - nxt_ruby_add_sptr(hash_env, NL("REQUEST_URI"), &r->target, + nxt_ruby_add_sptr(hash_env, nxt_rb_request_uri_str, &r->target, r->target_length); - nxt_ruby_add_sptr(hash_env, NL("PATH_INFO"), &r->path, r->path_length); - nxt_ruby_add_sptr(hash_env, NL("QUERY_STRING"), &r->query, + nxt_ruby_add_sptr(hash_env, nxt_rb_path_info_str, &r->path, r->path_length); + nxt_ruby_add_sptr(hash_env, nxt_rb_query_string_str, &r->query, r->query_length); - nxt_ruby_add_sptr(hash_env, NL("SERVER_PROTOCOL"), &r->version, + nxt_ruby_add_sptr(hash_env, nxt_rb_server_protocol_str, &r->version, r->version_length); - nxt_ruby_add_sptr(hash_env, NL("REMOTE_ADDR"), &r->remote, + nxt_ruby_add_sptr(hash_env, nxt_rb_remote_addr_str, &r->remote, r->remote_length); - nxt_ruby_add_sptr(hash_env, NL("SERVER_ADDR"), &r->local, r->local_length); - - nxt_ruby_add_sptr(hash_env, NL("SERVER_NAME"), &r->server_name, + nxt_ruby_add_sptr(hash_env, nxt_rb_server_addr_str, &r->local, + r->local_length); + nxt_ruby_add_sptr(hash_env, nxt_rb_server_name_str, &r->server_name, r->server_name_length); - nxt_ruby_add_str(hash_env, NL("SERVER_PORT"), "80", 2); - rb_hash_aset(hash_env, rb_str_new2("rack.url_scheme"), - r->tls ? rb_str_new2("https") : rb_str_new2("http")); + rb_hash_aset(hash_env, nxt_rb_server_port_str, nxt_rb_80_str); + + rb_hash_aset(hash_env, nxt_rb_rack_url_scheme_str, + r->tls ? nxt_rb_https_str : nxt_rb_http_str); for (i = 0; i < r->fields_count; i++) { f = r->fields + i; - nxt_ruby_add_sptr(hash_env, nxt_unit_sptr_get(&f->name), f->name_length, - &f->value, f->value_length); + name = rb_str_new(nxt_unit_sptr_get(&f->name), f->name_length); + + nxt_ruby_add_sptr(hash_env, name, &f->value, f->value_length); } if (r->content_length_field != NXT_UNIT_NONE_FIELD) { f = r->fields + r->content_length_field; - nxt_ruby_add_sptr(hash_env, NL("CONTENT_LENGTH"), + nxt_ruby_add_sptr(hash_env, nxt_rb_content_length_str, &f->value, f->value_length); } if (r->content_type_field != NXT_UNIT_NONE_FIELD) { f = r->fields + r->content_type_field; - nxt_ruby_add_sptr(hash_env, NL("CONTENT_TYPE"), + nxt_ruby_add_sptr(hash_env, nxt_rb_content_type_str, &f->value, f->value_length); } -#undef NL - return NXT_UNIT_OK; } nxt_inline void -nxt_ruby_add_sptr(VALUE hash_env, - const char *name, uint32_t name_len, nxt_unit_sptr_t *sptr, uint32_t len) +nxt_ruby_add_sptr(VALUE hash_env, VALUE name, + nxt_unit_sptr_t *sptr, uint32_t len) { char *str; str = nxt_unit_sptr_get(sptr); - rb_hash_aset(hash_env, rb_str_new(name, name_len), rb_str_new(str, len)); -} - - -nxt_inline void -nxt_ruby_add_str(VALUE hash_env, - const char *name, uint32_t name_len, const char *str, uint32_t len) -{ - rb_hash_aset(hash_env, rb_str_new(name, name_len), rb_str_new(str, len)); + rb_hash_aset(hash_env, name, rb_str_new(str, len)); } @@ -901,5 +971,7 @@ nxt_ruby_atexit(void) rb_gc_unregister_address(&nxt_ruby_call); rb_gc_unregister_address(&nxt_ruby_env); + nxt_ruby_done_strings(); + ruby_cleanup(0); } -- cgit From b6475df79cd14d80c794abfc9d1edbcebbe86f2c Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 5 Nov 2020 12:45:10 +0300 Subject: Ruby: request processing in multiple threads. This closes #482 issue on GitHub. --- src/nxt_application.h | 1 + src/nxt_conf_validation.c | 4 + src/nxt_main_process.c | 5 + src/ruby/nxt_ruby.c | 647 ++++++++++++++++++++++++++++++------------ src/ruby/nxt_ruby.h | 8 +- src/ruby/nxt_ruby_stream_io.c | 51 ++-- 6 files changed, 510 insertions(+), 206 deletions(-) diff --git a/src/nxt_application.h b/src/nxt_application.h index 08de0fce..b8e18a23 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -69,6 +69,7 @@ typedef struct { typedef struct { nxt_str_t script; + uint32_t threads; } nxt_ruby_app_conf_t; diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 1ce2d9ea..8dd9082d 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -615,6 +615,10 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_ruby_members[] = { .name = nxt_string("script"), .type = NXT_CONF_VLDT_STRING, .flags = NXT_CONF_VLDT_REQUIRED, + }, { + .name = nxt_string("threads"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_threads, }, NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index ce8d6916..631b3146 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -242,6 +242,11 @@ static nxt_conf_map_t nxt_ruby_app_conf[] = { NXT_CONF_MAP_STR, offsetof(nxt_common_app_conf_t, u.ruby.script), }, + { + nxt_string("threads"), + NXT_CONF_MAP_INT32, + offsetof(nxt_common_app_conf_t, u.ruby.threads), + }, }; diff --git a/src/ruby/nxt_ruby.c b/src/ruby/nxt_ruby.c index 7b383557..ffc1469a 100644 --- a/src/ruby/nxt_ruby.c +++ b/src/ruby/nxt_ruby.c @@ -8,6 +8,8 @@ #include #include +#include + #include NXT_RUBY_MOUNTS_H @@ -16,16 +18,15 @@ typedef struct { - nxt_task_t *task; - nxt_str_t *script; - VALUE builder; + nxt_task_t *task; + nxt_str_t *script; + nxt_ruby_ctx_t *rctx; } nxt_ruby_rack_init_t; static nxt_int_t nxt_ruby_start(nxt_task_t *task, nxt_process_data_t *data); static VALUE nxt_ruby_init_basic(VALUE arg); -static nxt_int_t nxt_ruby_init_io(nxt_task_t *task); static VALUE nxt_ruby_rack_init(nxt_ruby_rack_init_t *rack_init); static VALUE nxt_ruby_require_rubygems(VALUE arg); @@ -33,24 +34,40 @@ static VALUE nxt_ruby_bundler_setup(VALUE arg); static VALUE nxt_ruby_require_rack(VALUE arg); static VALUE nxt_ruby_rack_parse_script(VALUE ctx); static VALUE nxt_ruby_rack_env_create(VALUE arg); +static int nxt_ruby_init_io(nxt_ruby_ctx_t *rctx); static void nxt_ruby_request_handler(nxt_unit_request_info_t *req); +static void *nxt_ruby_request_handler_gvl(void *req); +static int nxt_ruby_ready_handler(nxt_unit_ctx_t *ctx); +static VALUE nxt_ruby_thread_func(VALUE arg); +static void *nxt_ruby_unit_run(void *ctx); +static void nxt_ruby_ubf(void *ctx); +static int nxt_ruby_init_threads(nxt_ruby_app_conf_t *c); +static void nxt_ruby_join_threads(nxt_unit_ctx_t *ctx, + nxt_ruby_app_conf_t *c); static VALUE nxt_ruby_rack_app_run(VALUE arg); -static int nxt_ruby_read_request(VALUE hash_env); +static int nxt_ruby_read_request(nxt_unit_request_info_t *req, VALUE hash_env); nxt_inline void nxt_ruby_add_sptr(VALUE hash_env, VALUE name, nxt_unit_sptr_t *sptr, uint32_t len); -static nxt_int_t nxt_ruby_rack_result_status(VALUE result); -static int nxt_ruby_rack_result_headers(VALUE result, nxt_int_t status); +static nxt_int_t nxt_ruby_rack_result_status(nxt_unit_request_info_t *req, + VALUE result); +static int nxt_ruby_rack_result_headers(nxt_unit_request_info_t *req, + VALUE result, nxt_int_t status); static int nxt_ruby_hash_info(VALUE r_key, VALUE r_value, VALUE arg); static int nxt_ruby_hash_add(VALUE r_key, VALUE r_value, VALUE arg); -static int nxt_ruby_rack_result_body(VALUE result); -static int nxt_ruby_rack_result_body_file_write(VALUE filepath); +static int nxt_ruby_rack_result_body(nxt_unit_request_info_t *req, + VALUE result); +static int nxt_ruby_rack_result_body_file_write(nxt_unit_request_info_t *req, + VALUE filepath); +static void *nxt_ruby_response_write_cb(void *read_info); static VALUE nxt_ruby_rack_result_body_each(VALUE body, VALUE arg, int argc, const VALUE *argv, VALUE blockarg); +static void *nxt_ruby_response_write(void *body); -static void nxt_ruby_exception_log(nxt_task_t *task, uint32_t level, - const char *desc); +static void nxt_ruby_exception_log(nxt_unit_request_info_t *req, + uint32_t level, const char *desc); +static void nxt_ruby_ctx_done(nxt_ruby_ctx_t *rctx); static void nxt_ruby_atexit(void); @@ -58,12 +75,11 @@ static uint32_t compat[] = { NXT_VERNUM, NXT_DEBUG, }; -static VALUE nxt_ruby_rackup; -static VALUE nxt_ruby_call; -static VALUE nxt_ruby_env; -static VALUE nxt_ruby_io_input; -static VALUE nxt_ruby_io_error; -static nxt_ruby_run_ctx_t nxt_ruby_run_ctx; +static VALUE nxt_ruby_rackup; +static VALUE nxt_ruby_call; + +static uint32_t nxt_ruby_threads; +static nxt_ruby_ctx_t *nxt_ruby_ctxs; NXT_EXPORT nxt_app_module_t nxt_app_module = { sizeof(compat), @@ -169,79 +185,114 @@ nxt_ruby_start(nxt_task_t *task, nxt_process_data_t *data) { int state, rc; VALUE res; + nxt_ruby_ctx_t ruby_ctx; nxt_unit_ctx_t *unit_ctx; nxt_unit_init_t ruby_unit_init; + nxt_ruby_app_conf_t *c; nxt_ruby_rack_init_t rack_init; nxt_common_app_conf_t *conf; static char *argv[2] = { (char *) "NGINX_Unit", (char *) "-e0" }; conf = data->app; + c = &conf->u.ruby; + + nxt_ruby_threads = c->threads; RUBY_INIT_STACK ruby_init(); ruby_options(2, argv); ruby_script("NGINX_Unit"); + ruby_ctx.env = Qnil; + ruby_ctx.io_input = Qnil; + ruby_ctx.io_error = Qnil; + ruby_ctx.thread = Qnil; + ruby_ctx.ctx = NULL; + ruby_ctx.req = NULL; + rack_init.task = task; - rack_init.script = &conf->u.ruby.script; + rack_init.script = &c->script; + rack_init.rctx = &ruby_ctx; nxt_ruby_init_strings(); res = rb_protect(nxt_ruby_init_basic, (VALUE) (uintptr_t) &rack_init, &state); if (nxt_slow_path(res == Qnil || state != 0)) { - nxt_ruby_exception_log(task, NXT_LOG_ALERT, + nxt_ruby_exception_log(NULL, NXT_LOG_ALERT, "Failed to init basic variables"); return NXT_ERROR; } + nxt_ruby_call = Qnil; + nxt_ruby_rackup = nxt_ruby_rack_init(&rack_init); if (nxt_slow_path(nxt_ruby_rackup == Qnil)) { return NXT_ERROR; } + rb_gc_register_address(&nxt_ruby_rackup); + nxt_ruby_call = rb_intern("call"); if (nxt_slow_path(nxt_ruby_call == Qnil)) { nxt_alert(task, "Ruby: Unable to find rack entry point"); - return NXT_ERROR; + goto fail; } - nxt_ruby_env = rb_protect(nxt_ruby_rack_env_create, Qnil, &state); - if (nxt_slow_path(state != 0)) { - nxt_ruby_exception_log(task, NXT_LOG_ALERT, + rb_gc_register_address(&nxt_ruby_call); + + ruby_ctx.env = rb_protect(nxt_ruby_rack_env_create, + (VALUE) (uintptr_t) &ruby_ctx, &state); + if (nxt_slow_path(ruby_ctx.env == Qnil || state != 0)) { + nxt_ruby_exception_log(NULL, NXT_LOG_ALERT, "Failed to create 'environ' variable"); - return NXT_ERROR; + goto fail; } - rb_gc_register_address(&nxt_ruby_rackup); - rb_gc_register_address(&nxt_ruby_call); - rb_gc_register_address(&nxt_ruby_env); + rc = nxt_ruby_init_threads(c); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + goto fail; + } nxt_unit_default_init(task, &ruby_unit_init); ruby_unit_init.callbacks.request_handler = nxt_ruby_request_handler; + ruby_unit_init.callbacks.ready_handler = nxt_ruby_ready_handler; ruby_unit_init.shm_limit = conf->shm_limit; + ruby_unit_init.data = c; + ruby_unit_init.ctx_data = &ruby_ctx; unit_ctx = nxt_unit_init(&ruby_unit_init); if (nxt_slow_path(unit_ctx == NULL)) { - return NXT_ERROR; + goto fail; } - nxt_ruby_run_ctx.unit_ctx = unit_ctx; + rc = (intptr_t) rb_thread_call_without_gvl(nxt_ruby_unit_run, unit_ctx, + nxt_ruby_ubf, unit_ctx); - rc = nxt_unit_run(unit_ctx); + nxt_ruby_join_threads(unit_ctx, c); - nxt_ruby_atexit(); + nxt_unit_done(unit_ctx); - nxt_ruby_run_ctx.unit_ctx = NULL; + nxt_ruby_ctx_done(&ruby_ctx); - nxt_unit_done(unit_ctx); + nxt_ruby_atexit(); exit(rc); return NXT_OK; + +fail: + + nxt_ruby_join_threads(NULL, c); + + nxt_ruby_ctx_done(&ruby_ctx); + + nxt_ruby_atexit(); + + return NXT_ERROR; } @@ -249,7 +300,6 @@ static VALUE nxt_ruby_init_basic(VALUE arg) { int state; - nxt_int_t rc; nxt_ruby_rack_init_t *rack_init; rack_init = (nxt_ruby_rack_init_t *) (uintptr_t) arg; @@ -265,56 +315,19 @@ nxt_ruby_init_basic(VALUE arg) rb_funcall(rb_cObject, rb_intern("require"), 1, rb_str_new2("enc/trans/transdb")); - rc = nxt_ruby_init_io(rack_init->task); - if (nxt_slow_path(rc != NXT_OK)) { - return Qnil; - } - return arg; } -static nxt_int_t -nxt_ruby_init_io(nxt_task_t *task) -{ - VALUE rb, io_input, io_error; - - io_input = nxt_ruby_stream_io_input_init(); - rb = Data_Wrap_Struct(io_input, 0, 0, &nxt_ruby_run_ctx); - - nxt_ruby_io_input = rb_funcall(io_input, rb_intern("new"), 1, rb); - if (nxt_slow_path(nxt_ruby_io_input == Qnil)) { - nxt_alert(task, "Ruby: Failed to create environment 'rack.input' var"); - - return NXT_ERROR; - } - - io_error = nxt_ruby_stream_io_error_init(); - rb = Data_Wrap_Struct(io_error, 0, 0, &nxt_ruby_run_ctx); - - nxt_ruby_io_error = rb_funcall(io_error, rb_intern("new"), 1, rb); - if (nxt_slow_path(nxt_ruby_io_error == Qnil)) { - nxt_alert(task, "Ruby: Failed to create environment 'rack.error' var"); - - return NXT_ERROR; - } - - rb_gc_register_address(&nxt_ruby_io_input); - rb_gc_register_address(&nxt_ruby_io_error); - - return NXT_OK; -} - - static VALUE nxt_ruby_rack_init(nxt_ruby_rack_init_t *rack_init) { int state; - VALUE rack, rackup, err; + VALUE rackup, err; rb_protect(nxt_ruby_require_rubygems, Qnil, &state); if (nxt_slow_path(state != 0)) { - nxt_ruby_exception_log(rack_init->task, NXT_LOG_ALERT, + nxt_ruby_exception_log(NULL, NXT_LOG_ALERT, "Failed to require 'rubygems' package"); return Qnil; } @@ -324,7 +337,7 @@ nxt_ruby_rack_init(nxt_ruby_rack_init_t *rack_init) err = rb_errinfo(); if (rb_obj_is_kind_of(err, rb_eLoadError) == Qfalse) { - nxt_ruby_exception_log(rack_init->task, NXT_LOG_ALERT, + nxt_ruby_exception_log(NULL, NXT_LOG_ALERT, "Failed to require 'bundler/setup' package"); return Qnil; } @@ -334,18 +347,15 @@ nxt_ruby_rack_init(nxt_ruby_rack_init_t *rack_init) rb_protect(nxt_ruby_require_rack, Qnil, &state); if (nxt_slow_path(state != 0)) { - nxt_ruby_exception_log(rack_init->task, NXT_LOG_ALERT, + nxt_ruby_exception_log(NULL, NXT_LOG_ALERT, "Failed to require 'rack' package"); return Qnil; } - rack = rb_const_get(rb_cObject, rb_intern("Rack")); - rack_init->builder = rb_const_get(rack, rb_intern("Builder")); - rackup = rb_protect(nxt_ruby_rack_parse_script, (VALUE) (uintptr_t) rack_init, &state); if (nxt_slow_path(TYPE(rackup) != T_ARRAY || state != 0)) { - nxt_ruby_exception_log(rack_init->task, NXT_LOG_ALERT, + nxt_ruby_exception_log(NULL, NXT_LOG_ALERT, "Failed to parse rack script"); return Qnil; } @@ -385,15 +395,18 @@ nxt_ruby_require_rack(VALUE arg) static VALUE nxt_ruby_rack_parse_script(VALUE ctx) { - VALUE script, res; + VALUE script, res, rack, builder; nxt_ruby_rack_init_t *rack_init; rack_init = (nxt_ruby_rack_init_t *) (uintptr_t) ctx; + rack = rb_const_get(rb_cObject, rb_intern("Rack")); + builder = rb_const_get(rack, rb_intern("Builder")); + script = rb_str_new((const char *) rack_init->script->start, (long) rack_init->script->length); - res = rb_funcall(rack_init->builder, rb_intern("parse_file"), 1, script); + res = rb_funcall(builder, rb_intern("parse_file"), 1, script); rb_str_free(script); @@ -404,7 +417,16 @@ nxt_ruby_rack_parse_script(VALUE ctx) static VALUE nxt_ruby_rack_env_create(VALUE arg) { - VALUE hash_env, version; + int rc; + VALUE hash_env, version; + nxt_ruby_ctx_t *rctx; + + rctx = (nxt_ruby_ctx_t *) (uintptr_t) arg; + + rc = nxt_ruby_init_io(rctx); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + return Qnil; + } hash_env = rb_hash_new(); @@ -418,47 +440,114 @@ nxt_ruby_rack_env_create(VALUE arg) rb_ary_push(version, UINT2NUM(NXT_RUBY_RACK_API_VERSION_MINOR)); rb_hash_aset(hash_env, rb_str_new2("rack.version"), version); - rb_hash_aset(hash_env, rb_str_new2("rack.input"), nxt_ruby_io_input); - rb_hash_aset(hash_env, rb_str_new2("rack.errors"), nxt_ruby_io_error); - rb_hash_aset(hash_env, rb_str_new2("rack.multithread"), Qfalse); + rb_hash_aset(hash_env, rb_str_new2("rack.input"), rctx->io_input); + rb_hash_aset(hash_env, rb_str_new2("rack.errors"), rctx->io_error); + rb_hash_aset(hash_env, rb_str_new2("rack.multithread"), + nxt_ruby_threads > 1 ? Qtrue : Qfalse); rb_hash_aset(hash_env, rb_str_new2("rack.multiprocess"), Qtrue); rb_hash_aset(hash_env, rb_str_new2("rack.run_once"), Qfalse); rb_hash_aset(hash_env, rb_str_new2("rack.hijack?"), Qfalse); rb_hash_aset(hash_env, rb_str_new2("rack.hijack"), Qnil); rb_hash_aset(hash_env, rb_str_new2("rack.hijack_io"), Qnil); + rctx->env = hash_env; + + rb_gc_register_address(&rctx->env); + return hash_env; } +static int +nxt_ruby_init_io(nxt_ruby_ctx_t *rctx) +{ + VALUE io_input, io_error; + + io_input = nxt_ruby_stream_io_input_init(); + + rctx->io_input = rb_funcall(io_input, rb_intern("new"), 1, + (VALUE) (uintptr_t) rctx); + if (nxt_slow_path(rctx->io_input == Qnil)) { + nxt_unit_alert(NULL, + "Ruby: Failed to create environment 'rack.input' var"); + + return NXT_UNIT_ERROR; + } + + rb_gc_register_address(&rctx->io_input); + + io_error = nxt_ruby_stream_io_error_init(); + + rctx->io_error = rb_funcall(io_error, rb_intern("new"), 1, + (VALUE) (uintptr_t) rctx); + if (nxt_slow_path(rctx->io_error == Qnil)) { + nxt_unit_alert(NULL, + "Ruby: Failed to create environment 'rack.error' var"); + + return NXT_UNIT_ERROR; + } + + rb_gc_register_address(&rctx->io_error); + + return NXT_UNIT_OK; +} + + static void nxt_ruby_request_handler(nxt_unit_request_info_t *req) { - int state; - VALUE res; + (void) rb_thread_call_with_gvl(nxt_ruby_request_handler_gvl, req); +} + + +static void * +nxt_ruby_request_handler_gvl(void *data) +{ + int state; + VALUE res; + nxt_ruby_ctx_t *rctx; + nxt_unit_request_info_t *req; + + req = data; - nxt_ruby_run_ctx.req = req; + rctx = req->ctx->data; + rctx->req = req; - res = rb_protect(nxt_ruby_rack_app_run, Qnil, &state); + res = rb_protect(nxt_ruby_rack_app_run, (VALUE) (uintptr_t) req, &state); if (nxt_slow_path(res == Qnil || state != 0)) { - nxt_ruby_exception_log(NULL, NXT_LOG_ERR, + nxt_ruby_exception_log(req, NXT_LOG_ERR, "Failed to run ruby script"); + + nxt_unit_request_done(req, NXT_UNIT_ERROR); + + } else { + nxt_unit_request_done(req, NXT_UNIT_OK); } + + rctx->req = NULL; + + return NULL; } static VALUE nxt_ruby_rack_app_run(VALUE arg) { - int rc; - VALUE env, result; - nxt_int_t status; + int rc; + VALUE env, result; + nxt_int_t status; + nxt_ruby_ctx_t *rctx; + nxt_unit_request_info_t *req; - env = rb_hash_dup(nxt_ruby_env); + req = (nxt_unit_request_info_t *) arg; - rc = nxt_ruby_read_request(env); + rctx = req->ctx->data; + + env = rb_hash_dup(rctx->env); + + rc = nxt_ruby_read_request(req, env); if (nxt_slow_path(rc != NXT_UNIT_OK)) { - nxt_unit_req_alert(nxt_ruby_run_ctx.req, + nxt_unit_req_alert(req, "Ruby: Failed to process incoming request"); goto fail; @@ -466,50 +555,44 @@ nxt_ruby_rack_app_run(VALUE arg) result = rb_funcall(nxt_ruby_rackup, nxt_ruby_call, 1, env); if (nxt_slow_path(TYPE(result) != T_ARRAY)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Invalid response format from application"); goto fail; } if (nxt_slow_path(RARRAY_LEN(result) != 3)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Invalid response format from application. " "Need 3 entries [Status, Headers, Body]"); goto fail; } - status = nxt_ruby_rack_result_status(result); + status = nxt_ruby_rack_result_status(req, result); if (nxt_slow_path(status < 0)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Invalid response status from application."); goto fail; } - rc = nxt_ruby_rack_result_headers(result, status); + rc = nxt_ruby_rack_result_headers(req, result, status); if (nxt_slow_path(rc != NXT_UNIT_OK)) { goto fail; } - rc = nxt_ruby_rack_result_body(result); + rc = nxt_ruby_rack_result_body(req, result); if (nxt_slow_path(rc != NXT_UNIT_OK)) { goto fail; } - nxt_unit_request_done(nxt_ruby_run_ctx.req, rc); - nxt_ruby_run_ctx.req = NULL; - rb_hash_delete(env, rb_obj_id(env)); return result; fail: - nxt_unit_request_done(nxt_ruby_run_ctx.req, NXT_UNIT_ERROR); - nxt_ruby_run_ctx.req = NULL; - rb_hash_delete(env, rb_obj_id(env)); return Qnil; @@ -517,14 +600,14 @@ fail: static int -nxt_ruby_read_request(VALUE hash_env) +nxt_ruby_read_request(nxt_unit_request_info_t *req, VALUE hash_env) { VALUE name; uint32_t i; nxt_unit_field_t *f; nxt_unit_request_t *r; - r = nxt_ruby_run_ctx.req->request; + r = req->request; nxt_ruby_add_sptr(hash_env, nxt_rb_request_method_str, &r->method, r->method_length); @@ -586,7 +669,7 @@ nxt_ruby_add_sptr(VALUE hash_env, VALUE name, static nxt_int_t -nxt_ruby_rack_result_status(VALUE result) +nxt_ruby_rack_result_status(nxt_unit_request_info_t *req, VALUE result) { VALUE status; @@ -601,7 +684,7 @@ nxt_ruby_rack_result_status(VALUE result) RSTRING_LEN(status)); } - nxt_unit_req_error(nxt_ruby_run_ctx.req, "Ruby: Invalid response 'status' " + nxt_unit_req_error(req, "Ruby: Invalid response 'status' " "format from application"); return -2; @@ -609,14 +692,16 @@ nxt_ruby_rack_result_status(VALUE result) typedef struct { - int rc; - uint32_t fields; - uint32_t size; + int rc; + uint32_t fields; + uint32_t size; + nxt_unit_request_info_t *req; } nxt_ruby_headers_info_t; static int -nxt_ruby_rack_result_headers(VALUE result, nxt_int_t status) +nxt_ruby_rack_result_headers(nxt_unit_request_info_t *req, VALUE result, + nxt_int_t status) { int rc; VALUE headers; @@ -624,7 +709,7 @@ nxt_ruby_rack_result_headers(VALUE result, nxt_int_t status) headers = rb_ary_entry(result, 1); if (nxt_slow_path(TYPE(headers) != T_HASH)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Invalid response 'headers' format from " "application"); @@ -636,6 +721,7 @@ nxt_ruby_rack_result_headers(VALUE result, nxt_int_t status) headers_info.rc = NXT_UNIT_OK; headers_info.fields = 0; headers_info.size = 0; + headers_info.req = req; rb_hash_foreach(headers, nxt_ruby_hash_info, (VALUE) (uintptr_t) &headers_info); @@ -643,13 +729,14 @@ nxt_ruby_rack_result_headers(VALUE result, nxt_int_t status) return headers_info.rc; } - rc = nxt_unit_response_init(nxt_ruby_run_ctx.req, status, + rc = nxt_unit_response_init(req, status, headers_info.fields, headers_info.size); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return rc; } - rb_hash_foreach(headers, nxt_ruby_hash_add, (VALUE) (uintptr_t) &rc); + rb_hash_foreach(headers, nxt_ruby_hash_add, + (VALUE) (uintptr_t) &headers_info); return rc; } @@ -664,14 +751,14 @@ nxt_ruby_hash_info(VALUE r_key, VALUE r_value, VALUE arg) headers_info = (void *) (uintptr_t) arg; if (nxt_slow_path(TYPE(r_key) != T_STRING)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(headers_info->req, "Ruby: Wrong header entry 'key' from application"); goto fail; } if (nxt_slow_path(TYPE(r_value) != T_STRING)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(headers_info->req, "Ruby: Wrong header entry 'value' from application"); goto fail; @@ -714,11 +801,13 @@ fail: static int nxt_ruby_hash_add(VALUE r_key, VALUE r_value, VALUE arg) { - int *rc; - uint32_t key_len; - const char *value, *value_end, *pos; + int *rc; + uint32_t key_len; + const char *value, *value_end, *pos; + nxt_ruby_headers_info_t *headers_info; - rc = (int *) (uintptr_t) arg; + headers_info = (void *) (uintptr_t) arg; + rc = &headers_info->rc; value = RSTRING_PTR(r_value); value_end = value + RSTRING_LEN(r_value); @@ -734,7 +823,7 @@ nxt_ruby_hash_add(VALUE r_key, VALUE r_value, VALUE arg) break; } - *rc = nxt_unit_response_add_field(nxt_ruby_run_ctx.req, + *rc = nxt_unit_response_add_field(headers_info->req, RSTRING_PTR(r_key), key_len, value, pos - value); if (nxt_slow_path(*rc != NXT_UNIT_OK)) { @@ -746,7 +835,7 @@ nxt_ruby_hash_add(VALUE r_key, VALUE r_value, VALUE arg) } if (value <= value_end) { - *rc = nxt_unit_response_add_field(nxt_ruby_run_ctx.req, + *rc = nxt_unit_response_add_field(headers_info->req, RSTRING_PTR(r_key), key_len, value, value_end - value); if (nxt_slow_path(*rc != NXT_UNIT_OK)) { @@ -765,7 +854,7 @@ fail: static int -nxt_ruby_rack_result_body(VALUE result) +nxt_ruby_rack_result_body(nxt_unit_request_info_t *req, VALUE result) { int rc; VALUE fn, body; @@ -776,24 +865,24 @@ nxt_ruby_rack_result_body(VALUE result) fn = rb_funcall(body, rb_intern("to_path"), 0); if (nxt_slow_path(TYPE(fn) != T_STRING)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Failed to get 'body' file path from " "application"); return NXT_UNIT_ERROR; } - rc = nxt_ruby_rack_result_body_file_write(fn); + rc = nxt_ruby_rack_result_body_file_write(req, fn); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return rc; } } else if (rb_respond_to(body, rb_intern("each"))) { rb_block_call(body, rb_intern("each"), 0, 0, - nxt_ruby_rack_result_body_each, 0); + nxt_ruby_rack_result_body_each, (VALUE) (uintptr_t) req); } else { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Invalid response 'body' format " "from application"); @@ -842,17 +931,24 @@ nxt_ruby_rack_file_read(nxt_unit_read_info_t *read_info, void *dst, size_t size) } +typedef struct { + nxt_unit_read_info_t read_info; + nxt_unit_request_info_t *req; +} nxt_ruby_read_info_t; + + static int -nxt_ruby_rack_result_body_file_write(VALUE filepath) +nxt_ruby_rack_result_body_file_write(nxt_unit_request_info_t *req, + VALUE filepath) { int fd, rc; struct stat finfo; nxt_ruby_rack_file_t ruby_file; - nxt_unit_read_info_t read_info; + nxt_ruby_read_info_t ri; fd = open(RSTRING_PTR(filepath), O_RDONLY, 0); if (nxt_slow_path(fd == -1)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Failed to open content file \"%s\": %s (%d)", RSTRING_PTR(filepath), strerror(errno), errno); @@ -861,7 +957,7 @@ nxt_ruby_rack_result_body_file_write(VALUE filepath) rc = fstat(fd, &finfo); if (nxt_slow_path(rc == -1)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Content file fstat(\"%s\") failed: %s (%d)", RSTRING_PTR(filepath), strerror(errno), errno); @@ -874,16 +970,16 @@ nxt_ruby_rack_result_body_file_write(VALUE filepath) ruby_file.pos = 0; ruby_file.rest = finfo.st_size; - read_info.read = nxt_ruby_rack_file_read; - read_info.eof = ruby_file.rest == 0; - read_info.buf_size = ruby_file.rest; - read_info.data = &ruby_file; + ri.read_info.read = nxt_ruby_rack_file_read; + ri.read_info.eof = ruby_file.rest == 0; + ri.read_info.buf_size = ruby_file.rest; + ri.read_info.data = &ruby_file; + ri.req = req; - rc = nxt_unit_response_write_cb(nxt_ruby_run_ctx.req, &read_info); - if (nxt_slow_path(rc != NXT_UNIT_OK)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, - "Ruby: Failed to write content file."); - } + rc = (intptr_t) rb_thread_call_without_gvl(nxt_ruby_response_write_cb, + &ri, + nxt_ruby_ubf, + req->ctx); close(fd); @@ -891,39 +987,77 @@ nxt_ruby_rack_result_body_file_write(VALUE filepath) } +static void * +nxt_ruby_response_write_cb(void *data) +{ + int rc; + nxt_ruby_read_info_t *ri; + + ri = data; + + rc = nxt_unit_response_write_cb(ri->req, &ri->read_info); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + nxt_unit_req_error(ri->req, "Ruby: Failed to write content file."); + } + + return (void *) (intptr_t) rc; +} + + +typedef struct { + VALUE body; + nxt_unit_request_info_t *req; +} nxt_ruby_write_info_t; + + static VALUE nxt_ruby_rack_result_body_each(VALUE body, VALUE arg, int argc, const VALUE *argv, VALUE blockarg) { - int rc; + nxt_ruby_write_info_t wi; if (TYPE(body) != T_STRING) { return Qnil; } - rc = nxt_unit_response_write(nxt_ruby_run_ctx.req, RSTRING_PTR(body), - RSTRING_LEN(body)); + wi.body = body; + wi.req = (void *) (uintptr_t) arg; + + (void) rb_thread_call_without_gvl(nxt_ruby_response_write, + (void *) (uintptr_t) &wi, + nxt_ruby_ubf, wi.req->ctx); + + return Qnil; +} + + +static void * +nxt_ruby_response_write(void *data) +{ + int rc; + nxt_ruby_write_info_t *wi; + + wi = data; + + rc = nxt_unit_response_write(wi->req, RSTRING_PTR(wi->body), + RSTRING_LEN(wi->body)); if (nxt_slow_path(rc != NXT_UNIT_OK)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(wi->req, "Ruby: Failed to write 'body' from application"); } - return Qnil; + return (void *) (intptr_t) rc; } static void -nxt_ruby_exception_log(nxt_task_t *task, uint32_t level, const char *desc) +nxt_ruby_exception_log(nxt_unit_request_info_t *req, uint32_t level, + const char *desc) { int i; VALUE err, ary, eclass, msg; - if (task != NULL) { - nxt_log(task, level, "Ruby: %s", desc); - - } else { - nxt_unit_log(nxt_ruby_run_ctx.unit_ctx, level, "Ruby: %s", desc); - } + nxt_unit_req_log(req, level, "Ruby: %s", desc); err = rb_errinfo(); if (nxt_slow_path(err == Qnil)) { @@ -938,25 +1072,30 @@ nxt_ruby_exception_log(nxt_task_t *task, uint32_t level, const char *desc) eclass = rb_class_name(rb_class_of(err)); msg = rb_funcall(err, rb_intern("message"), 0); - if (task != NULL) { - nxt_log(task, level, "Ruby: %s: %s (%s)", - RSTRING_PTR(RARRAY_PTR(ary)[0]), - RSTRING_PTR(msg), RSTRING_PTR(eclass)); - - } else { - nxt_unit_log(nxt_ruby_run_ctx.unit_ctx, level, "Ruby: %s: %s (%s)", + nxt_unit_req_log(req, level, "Ruby: %s: %s (%s)", RSTRING_PTR(RARRAY_PTR(ary)[0]), RSTRING_PTR(msg), RSTRING_PTR(eclass)); - } for (i = 1; i < RARRAY_LEN(ary); i++) { - if (task != NULL) { - nxt_log(task, level, "from %s", RSTRING_PTR(RARRAY_PTR(ary)[i])); - - } else { - nxt_unit_log(nxt_ruby_run_ctx.unit_ctx, level, "from %s", + nxt_unit_req_log(req, level, "from %s", RSTRING_PTR(RARRAY_PTR(ary)[i])); - } + } +} + + +static void +nxt_ruby_ctx_done(nxt_ruby_ctx_t *rctx) +{ + if (rctx->io_input != Qnil) { + rb_gc_unregister_address(&rctx->io_input); + } + + if (rctx->io_error != Qnil) { + rb_gc_unregister_address(&rctx->io_error); + } + + if (rctx->env != Qnil) { + rb_gc_unregister_address(&rctx->env); } } @@ -964,14 +1103,170 @@ nxt_ruby_exception_log(nxt_task_t *task, uint32_t level, const char *desc) static void nxt_ruby_atexit(void) { - rb_gc_unregister_address(&nxt_ruby_io_input); - rb_gc_unregister_address(&nxt_ruby_io_error); + if (nxt_ruby_rackup != Qnil) { + rb_gc_unregister_address(&nxt_ruby_rackup); + } - rb_gc_unregister_address(&nxt_ruby_rackup); - rb_gc_unregister_address(&nxt_ruby_call); - rb_gc_unregister_address(&nxt_ruby_env); + if (nxt_ruby_call != Qnil) { + rb_gc_unregister_address(&nxt_ruby_call); + } nxt_ruby_done_strings(); ruby_cleanup(0); } + + +static int +nxt_ruby_ready_handler(nxt_unit_ctx_t *ctx) +{ + VALUE res; + uint32_t i; + nxt_ruby_ctx_t *rctx; + nxt_ruby_app_conf_t *c; + + /* Worker thread context. */ + if (!nxt_unit_is_main_ctx(ctx)) { + return NXT_UNIT_OK; + } + + c = ctx->unit->data; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + for (i = 0; i < c->threads - 1; i++) { + rctx = &nxt_ruby_ctxs[i]; + + rctx->ctx = ctx; + + res = rb_thread_create(RUBY_METHOD_FUNC(nxt_ruby_thread_func), rctx); + + if (nxt_fast_path(res != Qnil)) { + nxt_unit_debug(ctx, "thread #%d created", (int) (i + 1)); + + rctx->thread = res; + + } else { + nxt_unit_alert(ctx, "thread #%d create failed", (int) (i + 1)); + } + } + + return NXT_UNIT_OK; +} + + +static VALUE +nxt_ruby_thread_func(VALUE arg) +{ + nxt_unit_ctx_t *ctx; + nxt_ruby_ctx_t *rctx; + + rctx = (nxt_ruby_ctx_t *) (uintptr_t) arg; + + nxt_unit_debug(rctx->ctx, "worker thread start"); + + ctx = nxt_unit_ctx_alloc(rctx->ctx, rctx); + if (nxt_slow_path(ctx == NULL)) { + goto fail; + } + + (void) rb_thread_call_without_gvl(nxt_ruby_unit_run, ctx, + nxt_ruby_ubf, ctx); + + nxt_unit_done(ctx); + +fail: + + nxt_unit_debug(NULL, "worker thread end"); + + return Qnil; +} + + +static void * +nxt_ruby_unit_run(void *ctx) +{ + return (void *) (intptr_t) nxt_unit_run(ctx); +} + + +static void +nxt_ruby_ubf(void *ctx) +{ + nxt_unit_warn(ctx, "Ruby: UBF"); +} + + +static int +nxt_ruby_init_threads(nxt_ruby_app_conf_t *c) +{ + int state; + uint32_t i; + nxt_ruby_ctx_t *rctx; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + nxt_ruby_ctxs = nxt_unit_malloc(NULL, sizeof(nxt_ruby_ctx_t) + * (c->threads - 1)); + if (nxt_slow_path(nxt_ruby_ctxs == NULL)) { + nxt_unit_alert(NULL, "Failed to allocate run contexts array"); + + return NXT_UNIT_ERROR; + } + + for (i = 0; i < c->threads - 1; i++) { + rctx = &nxt_ruby_ctxs[i]; + + rctx->env = Qnil; + rctx->io_input = Qnil; + rctx->io_error = Qnil; + rctx->thread = Qnil; + } + + for (i = 0; i < c->threads - 1; i++) { + rctx = &nxt_ruby_ctxs[i]; + + rctx->env = rb_protect(nxt_ruby_rack_env_create, + (VALUE) (uintptr_t) rctx, &state); + if (nxt_slow_path(rctx->env == Qnil || state != 0)) { + nxt_ruby_exception_log(NULL, NXT_LOG_ALERT, + "Failed to create 'environ' variable"); + return NXT_UNIT_ERROR; + } + } + + return NXT_UNIT_OK; +} + + +static void +nxt_ruby_join_threads(nxt_unit_ctx_t *ctx, nxt_ruby_app_conf_t *c) +{ + uint32_t i; + nxt_ruby_ctx_t *rctx; + + if (nxt_ruby_ctxs == NULL) { + return; + } + + for(i = 0; i < c->threads - 1; i++) { + rctx = &nxt_ruby_ctxs[i]; + + if (rctx->thread != Qnil) { + rb_funcall(rctx->thread, rb_intern("join"), 0); + + nxt_unit_debug(ctx, "thread #%d joined", (int) (i + 1)); + + } else { + nxt_unit_debug(ctx, "thread #%d not started", (int) (i + 1)); + } + + nxt_ruby_ctx_done(rctx); + } + + nxt_unit_free(ctx, nxt_ruby_ctxs); +} diff --git a/src/ruby/nxt_ruby.h b/src/ruby/nxt_ruby.h index 77a65894..26430021 100644 --- a/src/ruby/nxt_ruby.h +++ b/src/ruby/nxt_ruby.h @@ -21,9 +21,13 @@ typedef struct { - nxt_unit_ctx_t *unit_ctx; + VALUE env; + VALUE io_input; + VALUE io_error; + VALUE thread; + nxt_unit_ctx_t *ctx; nxt_unit_request_info_t *req; -} nxt_ruby_run_ctx_t; +} nxt_ruby_ctx_t; VALUE nxt_ruby_stream_io_input_init(void); diff --git a/src/ruby/nxt_ruby_stream_io.c b/src/ruby/nxt_ruby_stream_io.c index cc110035..69bf289e 100644 --- a/src/ruby/nxt_ruby_stream_io.c +++ b/src/ruby/nxt_ruby_stream_io.c @@ -8,7 +8,7 @@ #include -static VALUE nxt_ruby_stream_io_new(VALUE class, VALUE wrap); +static VALUE nxt_ruby_stream_io_new(VALUE class, VALUE arg); static VALUE nxt_ruby_stream_io_initialize(int argc, VALUE *argv, VALUE self); static VALUE nxt_ruby_stream_io_gets(VALUE obj); static VALUE nxt_ruby_stream_io_each(VALUE obj); @@ -16,8 +16,7 @@ static VALUE nxt_ruby_stream_io_read(VALUE obj, VALUE args); static VALUE nxt_ruby_stream_io_rewind(VALUE obj); static VALUE nxt_ruby_stream_io_puts(VALUE obj, VALUE args); static VALUE nxt_ruby_stream_io_write(VALUE obj, VALUE args); -nxt_inline long nxt_ruby_stream_io_s_write(nxt_ruby_run_ctx_t *run_ctx, - VALUE val); +nxt_inline long nxt_ruby_stream_io_s_write(nxt_ruby_ctx_t *rctx, VALUE val); static VALUE nxt_ruby_stream_io_flush(VALUE obj); @@ -63,13 +62,11 @@ nxt_ruby_stream_io_error_init(void) static VALUE -nxt_ruby_stream_io_new(VALUE class, VALUE wrap) +nxt_ruby_stream_io_new(VALUE class, VALUE arg) { - VALUE self; - nxt_ruby_run_ctx_t *run_ctx; + VALUE self; - Data_Get_Struct(wrap, nxt_ruby_run_ctx_t, run_ctx); - self = Data_Wrap_Struct(class, 0, 0, run_ctx); + self = Data_Wrap_Struct(class, 0, 0, (void *) (uintptr_t) arg); rb_obj_call_init(self, 0, NULL); @@ -89,12 +86,11 @@ nxt_ruby_stream_io_gets(VALUE obj) { VALUE buf; ssize_t res; - nxt_ruby_run_ctx_t *run_ctx; + nxt_ruby_ctx_t *rctx; nxt_unit_request_info_t *req; - Data_Get_Struct(obj, nxt_ruby_run_ctx_t, run_ctx); - - req = run_ctx->req; + Data_Get_Struct(obj, nxt_ruby_ctx_t, rctx); + req = rctx->req; if (req->content_length == 0) { return Qnil; @@ -145,13 +141,13 @@ nxt_ruby_stream_io_each(VALUE obj) static VALUE nxt_ruby_stream_io_read(VALUE obj, VALUE args) { - VALUE buf; - long copy_size, u_size; - nxt_ruby_run_ctx_t *run_ctx; + VALUE buf; + long copy_size, u_size; + nxt_ruby_ctx_t *rctx; - Data_Get_Struct(obj, nxt_ruby_run_ctx_t, run_ctx); + Data_Get_Struct(obj, nxt_ruby_ctx_t, rctx); - copy_size = run_ctx->req->content_length; + copy_size = rctx->req->content_length; if (RARRAY_LEN(args) > 0 && TYPE(RARRAY_PTR(args)[0]) == T_FIXNUM) { u_size = NUM2LONG(RARRAY_PTR(args)[0]); @@ -175,8 +171,7 @@ nxt_ruby_stream_io_read(VALUE obj, VALUE args) return Qnil; } - copy_size = nxt_unit_request_read(run_ctx->req, RSTRING_PTR(buf), - copy_size); + copy_size = nxt_unit_request_read(rctx->req, RSTRING_PTR(buf), copy_size); if (RARRAY_LEN(args) > 1 && TYPE(RARRAY_PTR(args)[1]) == T_STRING) { @@ -200,15 +195,15 @@ nxt_ruby_stream_io_rewind(VALUE obj) static VALUE nxt_ruby_stream_io_puts(VALUE obj, VALUE args) { - nxt_ruby_run_ctx_t *run_ctx; + nxt_ruby_ctx_t *rctx; if (RARRAY_LEN(args) != 1) { return Qnil; } - Data_Get_Struct(obj, nxt_ruby_run_ctx_t, run_ctx); + Data_Get_Struct(obj, nxt_ruby_ctx_t, rctx); - nxt_ruby_stream_io_s_write(run_ctx, RARRAY_PTR(args)[0]); + nxt_ruby_stream_io_s_write(rctx, RARRAY_PTR(args)[0]); return Qnil; } @@ -217,23 +212,23 @@ nxt_ruby_stream_io_puts(VALUE obj, VALUE args) static VALUE nxt_ruby_stream_io_write(VALUE obj, VALUE args) { - long len; - nxt_ruby_run_ctx_t *run_ctx; + long len; + nxt_ruby_ctx_t *rctx; if (RARRAY_LEN(args) != 1) { return Qnil; } - Data_Get_Struct(obj, nxt_ruby_run_ctx_t, run_ctx); + Data_Get_Struct(obj, nxt_ruby_ctx_t, rctx); - len = nxt_ruby_stream_io_s_write(run_ctx, RARRAY_PTR(args)[0]); + len = nxt_ruby_stream_io_s_write(rctx, RARRAY_PTR(args)[0]); return LONG2FIX(len); } nxt_inline long -nxt_ruby_stream_io_s_write(nxt_ruby_run_ctx_t *run_ctx, VALUE val) +nxt_ruby_stream_io_s_write(nxt_ruby_ctx_t *rctx, VALUE val) { if (nxt_slow_path(val == Qnil)) { return 0; @@ -247,7 +242,7 @@ nxt_ruby_stream_io_s_write(nxt_ruby_run_ctx_t *run_ctx, VALUE val) } } - nxt_unit_req_error(run_ctx->req, "Ruby: %s", RSTRING_PTR(val)); + nxt_unit_req_error(rctx->req, "Ruby: %s", RSTRING_PTR(val)); return RSTRING_LEN(val); } -- cgit From e17e73eddab015b8942d2228780bbd9afcc5e392 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 5 Nov 2020 12:45:26 +0300 Subject: Tests: added Ruby threading tests. --- test/ruby/threads/config.ru | 13 +++++++++++++ test/test_ruby_application.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 test/ruby/threads/config.ru diff --git a/test/ruby/threads/config.ru b/test/ruby/threads/config.ru new file mode 100644 index 00000000..2a234d0d --- /dev/null +++ b/test/ruby/threads/config.ru @@ -0,0 +1,13 @@ +app = Proc.new do |env| + delay = env['HTTP_X_DELAY'].to_f + + sleep(delay) + + ['200', { + 'Content-Length' => 0.to_s, + 'Rack-Multithread' => env['rack.multithread'].to_s, + 'X-Thread' => Thread.current.object_id.to_s + }, []] +end + +run app diff --git a/test/test_ruby_application.py b/test/test_ruby_application.py index b765f390..e42fb97f 100644 --- a/test/test_ruby_application.py +++ b/test/test_ruby_application.py @@ -351,3 +351,44 @@ class TestRubyApplication(TestApplicationRuby): assert len(headers['X-Release-Date']) > 0, 'RUBY_RELEASE_DATE' assert len(headers['X-Revision']) > 0, 'RUBY_REVISION' assert len(headers['X-Version']) > 0, 'RUBY_VERSION' + + def test_ruby_application_threads(self): + self.load('threads') + + assert 'success' in self.conf( + '4', 'applications/threads/threads' + ), 'configure 4 threads' + + socks = [] + + for i in range(4): + (_, sock) = self.get( + headers={ + 'Host': 'localhost', + 'X-Delay': '2', + 'Connection': 'close', + }, + no_recv=True, + start=True, + ) + + socks.append(sock) + + threads = set() + + for sock in socks: + resp = self.recvall(sock).decode('utf-8') + + self.log_in(resp) + + resp = self._resp_to_dict(resp) + + assert resp['status'] == 200, 'status' + + threads.add(resp['headers']['X-Thread']) + + assert resp['headers']['Rack-Multithread'] == 'true', 'multithread' + + sock.close() + + assert len(socks) == len(threads), 'threads differs' -- cgit From d321d454f9304b083d62280d0621f282a74047ee Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 5 Nov 2020 16:10:59 +0300 Subject: Perl: request processing in multiple threads. This closes #486 issue on GitHub. --- src/nxt_application.h | 2 + src/nxt_conf_validation.c | 8 + src/nxt_main_process.c | 12 + src/perl/nxt_perl_psgi.c | 494 +++++++++++++++++++++++++++++------------ src/perl/nxt_perl_psgi_layer.h | 6 +- 5 files changed, 374 insertions(+), 148 deletions(-) diff --git a/src/nxt_application.h b/src/nxt_application.h index b8e18a23..66c0e1f7 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -64,6 +64,8 @@ typedef struct { typedef struct { char *script; + uint32_t threads; + uint32_t thread_stack_size; } nxt_perl_app_conf_t; diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 8dd9082d..8e6279fa 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -604,6 +604,14 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_perl_members[] = { .name = nxt_string("script"), .type = NXT_CONF_VLDT_STRING, .flags = NXT_CONF_VLDT_REQUIRED, + }, { + .name = nxt_string("threads"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_threads, + }, { + .name = nxt_string("thread_stack_size"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_thread_stack_size, }, NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 631b3146..99fc7995 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -233,6 +233,18 @@ static nxt_conf_map_t nxt_perl_app_conf[] = { NXT_CONF_MAP_CSTRZ, offsetof(nxt_common_app_conf_t, u.perl.script), }, + + { + nxt_string("threads"), + NXT_CONF_MAP_INT32, + offsetof(nxt_common_app_conf_t, u.perl.threads), + }, + + { + nxt_string("thread_stack_size"), + NXT_CONF_MAP_INT32, + offsetof(nxt_common_app_conf_t, u.perl.thread_stack_size), + }, }; diff --git a/src/perl/nxt_perl_psgi.c b/src/perl/nxt_perl_psgi.c index 16079a38..5df1465d 100644 --- a/src/perl/nxt_perl_psgi.c +++ b/src/perl/nxt_perl_psgi.c @@ -18,14 +18,14 @@ typedef struct { PerlInterpreter *my_perl; - nxt_unit_request_info_t *req; -} nxt_perl_psgi_input_t; - - -typedef struct { - PerlInterpreter *my_perl; + nxt_perl_psgi_io_arg_t arg_input; + nxt_perl_psgi_io_arg_t arg_error; SV *app; -} nxt_perl_psgi_module_t; + CV *cb; + nxt_unit_request_info_t *req; + pthread_t thread; + nxt_unit_ctx_t *ctx; +} nxt_perl_psgi_ctx_t; static long nxt_perl_psgi_io_input_read(PerlInterpreter *my_perl, @@ -62,11 +62,11 @@ static nxt_int_t nxt_perl_psgi_io_input_init(PerlInterpreter *my_perl, static nxt_int_t nxt_perl_psgi_io_error_init(PerlInterpreter *my_perl, nxt_perl_psgi_io_arg_t *arg); -static PerlInterpreter *nxt_perl_psgi_interpreter_init(nxt_task_t *task, - char *script, SV **app); +static int nxt_perl_psgi_ctx_init(const char *script, + nxt_perl_psgi_ctx_t *pctx); static SV *nxt_perl_psgi_env_create(PerlInterpreter *my_perl, - nxt_unit_request_info_t *req, nxt_perl_psgi_input_t *input); + nxt_unit_request_info_t *req); nxt_inline int nxt_perl_psgi_add_sptr(PerlInterpreter *my_perl, HV *hash_env, const char *name, uint32_t name_len, nxt_unit_sptr_t *sptr, uint32_t len); nxt_inline int nxt_perl_psgi_add_str(PerlInterpreter *my_perl, HV *hash_env, @@ -75,8 +75,7 @@ nxt_inline int nxt_perl_psgi_add_value(PerlInterpreter *my_perl, HV *hash_env, const char *name, uint32_t name_len, void *value); -static u_char *nxt_perl_psgi_module_create(nxt_task_t *task, - const char *script); +static char *nxt_perl_psgi_module_create(const char *script); static nxt_int_t nxt_perl_psgi_result_status(PerlInterpreter *my_perl, SV *result); @@ -96,18 +95,20 @@ static void nxt_perl_psgi_result_cb(PerlInterpreter *my_perl, SV *result, nxt_unit_request_info_t *req); static nxt_int_t nxt_perl_psgi_start(nxt_task_t *task, - nxt_process_data_t *conf); + nxt_process_data_t *data); static void nxt_perl_psgi_request_handler(nxt_unit_request_info_t *req); -static void nxt_perl_psgi_atexit(void); - -typedef SV *(*nxt_perl_psgi_callback_f)(PerlInterpreter *my_perl, - SV *env, nxt_task_t *task); - -static CV *nxt_perl_psgi_cb; -static PerlInterpreter *nxt_perl_psgi; -static nxt_perl_psgi_io_arg_t nxt_perl_psgi_arg_input; -static nxt_perl_psgi_io_arg_t nxt_perl_psgi_arg_error; -static nxt_unit_request_info_t *nxt_perl_psgi_request; +static int nxt_perl_psgi_ready_handler(nxt_unit_ctx_t *ctx); +static void *nxt_perl_psgi_thread_func(void *main_ctx); +static int nxt_perl_psgi_init_threads(nxt_perl_app_conf_t *c); +static void nxt_perl_psgi_join_threads(nxt_unit_ctx_t *ctx, + nxt_perl_app_conf_t *c); +static void nxt_perl_psgi_ctx_free(nxt_perl_psgi_ctx_t *pctx); + +static CV *nxt_perl_psgi_write; +static CV *nxt_perl_psgi_close; +static CV *nxt_perl_psgi_cb; +static pthread_attr_t *nxt_perl_psgi_thread_attr; +static nxt_perl_psgi_ctx_t *nxt_perl_psgi_ctxs; static uint32_t nxt_perl_psgi_compat[] = { NXT_VERNUM, NXT_DEBUG, @@ -129,11 +130,11 @@ static long nxt_perl_psgi_io_input_read(PerlInterpreter *my_perl, nxt_perl_psgi_io_arg_t *arg, void *vbuf, size_t length) { - nxt_perl_psgi_input_t *input; + nxt_perl_psgi_ctx_t *pctx; - input = (nxt_perl_psgi_input_t *) arg->ctx; + pctx = arg->pctx; - return nxt_unit_request_read(input->req, vbuf, length); + return nxt_unit_request_read(pctx->req, vbuf, length); } @@ -165,10 +166,11 @@ static long nxt_perl_psgi_io_error_write(PerlInterpreter *my_perl, nxt_perl_psgi_io_arg_t *arg, const void *vbuf, size_t length) { - nxt_perl_psgi_input_t *input; + nxt_perl_psgi_ctx_t *pctx; - input = (nxt_perl_psgi_input_t *) arg->ctx; - nxt_unit_req_error(input->req, "Perl: %s", (const char*) vbuf); + pctx = arg->pctx; + + nxt_unit_req_error(pctx->req, "Perl: %s", (const char*) vbuf); return (long) length; } @@ -216,9 +218,10 @@ XS(XS_NGINX__Unit__PSGI_exit) XS(XS_NGINX__Unit__Sandbox_write); XS(XS_NGINX__Unit__Sandbox_write) { - int rc; - char *body; - size_t len; + int rc; + char *body; + size_t len; + nxt_perl_psgi_ctx_t *pctx; dXSARGS; @@ -230,7 +233,9 @@ XS(XS_NGINX__Unit__Sandbox_write) body = SvPV(ST(1), len); - rc = nxt_unit_response_write(nxt_perl_psgi_request, body, len); + pctx = CvXSUBANY(cv).any_ptr; + + rc = nxt_unit_response_write(pctx->req, body, len); if (nxt_slow_path(rc != NXT_UNIT_OK)) { Perl_croak(aTHX_ "Failed to write response body"); @@ -242,15 +247,11 @@ XS(XS_NGINX__Unit__Sandbox_write) nxt_inline void -nxt_perl_psgi_cb_request_done(nxt_int_t status) +nxt_perl_psgi_cb_request_done(nxt_perl_psgi_ctx_t *pctx, int status) { - nxt_unit_request_info_t *req; - - req = nxt_perl_psgi_request; - - if (req != NULL) { - nxt_unit_request_done(req, status); - nxt_perl_psgi_request = NULL; + if (pctx->req != NULL) { + nxt_unit_request_done(pctx->req, status); + pctx->req = NULL; } } @@ -262,7 +263,7 @@ XS(XS_NGINX__Unit__Sandbox_close) ax = POPMARK; - nxt_perl_psgi_cb_request_done(NXT_UNIT_OK); + nxt_perl_psgi_cb_request_done(CvXSUBANY(cv).any_ptr, NXT_UNIT_OK); XSRETURN_NO; } @@ -271,14 +272,15 @@ XS(XS_NGINX__Unit__Sandbox_close) XS(XS_NGINX__Unit__Sandbox_cb); XS(XS_NGINX__Unit__Sandbox_cb) { - SV *obj; - int rc; - long array_len; + SV *obj; + int rc; + long array_len; + nxt_perl_psgi_ctx_t *pctx; dXSARGS; if (nxt_slow_path(items != 1)) { - nxt_perl_psgi_cb_request_done(NXT_UNIT_ERROR); + nxt_perl_psgi_cb_request_done(CvXSUBANY(cv).any_ptr, NXT_UNIT_ERROR); Perl_croak(aTHX_ "Wrong number of arguments"); @@ -288,7 +290,7 @@ XS(XS_NGINX__Unit__Sandbox_cb) if (nxt_slow_path(SvOK(ST(0)) == 0 || SvROK(ST(0)) == 0 || SvTYPE(SvRV(ST(0))) != SVt_PVAV)) { - nxt_perl_psgi_cb_request_done(NXT_UNIT_ERROR); + nxt_perl_psgi_cb_request_done(CvXSUBANY(cv).any_ptr, NXT_UNIT_ERROR); Perl_croak(aTHX_ "PSGI: An unexpected response was received " "from Perl Application"); @@ -296,10 +298,11 @@ XS(XS_NGINX__Unit__Sandbox_cb) XSRETURN_EMPTY; } - rc = nxt_perl_psgi_result_array(PERL_GET_CONTEXT, ST(0), - nxt_perl_psgi_request); + pctx = CvXSUBANY(cv).any_ptr; + + rc = nxt_perl_psgi_result_array(PERL_GET_CONTEXT, ST(0), pctx->req); if (nxt_slow_path(rc != NXT_UNIT_OK)) { - nxt_perl_psgi_cb_request_done(NXT_UNIT_ERROR); + nxt_perl_psgi_cb_request_done(CvXSUBANY(cv).any_ptr, NXT_UNIT_ERROR); Perl_croak(aTHX_ (char *) NULL); @@ -316,7 +319,7 @@ XS(XS_NGINX__Unit__Sandbox_cb) XSRETURN(1); } - nxt_perl_psgi_cb_request_done(NXT_UNIT_OK); + nxt_perl_psgi_cb_request_done(CvXSUBANY(cv).any_ptr, NXT_UNIT_OK); XSRETURN_EMPTY; } @@ -335,10 +338,11 @@ nxt_perl_psgi_xs_init(pTHX) /* DynaLoader for Perl modules who use XS */ newXS("DynaLoader::boot_DynaLoader", boot_DynaLoader, __FILE__); - newXS("NGINX::Unit::Sandbox::write", XS_NGINX__Unit__Sandbox_write, - __FILE__); - newXS("NGINX::Unit::Sandbox::close", XS_NGINX__Unit__Sandbox_close, - __FILE__); + nxt_perl_psgi_write = newXS("NGINX::Unit::Sandbox::write", + XS_NGINX__Unit__Sandbox_write, __FILE__); + + nxt_perl_psgi_close = newXS("NGINX::Unit::Sandbox::close", + XS_NGINX__Unit__Sandbox_close, __FILE__); nxt_perl_psgi_cb = newXS("NGINX::Unit::Sandbox::cb", XS_NGINX__Unit__Sandbox_cb, __FILE__); @@ -416,10 +420,10 @@ nxt_perl_psgi_call_method(PerlInterpreter *my_perl, SV *obj, const char *method, } -static u_char * -nxt_perl_psgi_module_create(nxt_task_t *task, const char *script) +static char * +nxt_perl_psgi_module_create(const char *script) { - u_char *buf, *p; + char *buf, *p; size_t length; static nxt_str_t prefix = nxt_string( @@ -441,12 +445,11 @@ nxt_perl_psgi_module_create(nxt_task_t *task, const char *script) length = strlen(script); - buf = nxt_malloc(prefix.length + length + suffix.length); - + buf = nxt_unit_malloc(NULL, prefix.length + length + suffix.length); if (nxt_slow_path(buf == NULL)) { - nxt_log_error(NXT_LOG_ERR, task->log, - "PSGI: Failed to allocate memory " - "for Perl script file %s", script); + nxt_unit_alert(NULL, "PSGI: Failed to allocate memory " + "for Perl script file %s", script); + return NULL; } @@ -518,30 +521,27 @@ nxt_perl_psgi_io_error_init(PerlInterpreter *my_perl, } -static PerlInterpreter * -nxt_perl_psgi_interpreter_init(nxt_task_t *task, char *script, SV **app) +static int +nxt_perl_psgi_ctx_init(const char *script, nxt_perl_psgi_ctx_t *pctx) { - int status, pargc; - char **pargv, **penv; - u_char *run_module; + int status; + char *run_module; PerlInterpreter *my_perl; static char argv[] = "\0""-e\0""0"; static char *embedding[] = { &argv[0], &argv[1], &argv[4] }; - pargc = 0; - pargv = NULL; - penv = NULL; - - PERL_SYS_INIT3(&pargc, &pargv, &penv); - my_perl = perl_alloc(); if (nxt_slow_path(my_perl == NULL)) { - nxt_alert(task, "PSGI: Failed to allocate memory for Perl interpreter"); - return NULL; + nxt_unit_alert(NULL, + "PSGI: Failed to allocate memory for Perl interpreter"); + + return NXT_UNIT_ERROR; } + pctx->my_perl = my_perl; + run_module = NULL; perl_construct(my_perl); @@ -550,77 +550,86 @@ nxt_perl_psgi_interpreter_init(nxt_task_t *task, char *script, SV **app) status = perl_parse(my_perl, nxt_perl_psgi_xs_init, 3, embedding, NULL); if (nxt_slow_path(status != 0)) { - nxt_alert(task, "PSGI: Failed to parse Perl Script"); + nxt_unit_alert(NULL, "PSGI: Failed to parse Perl Script"); goto fail; } + CvXSUBANY(nxt_perl_psgi_write).any_ptr = pctx; + CvXSUBANY(nxt_perl_psgi_close).any_ptr = pctx; + CvXSUBANY(nxt_perl_psgi_cb).any_ptr = pctx; + + pctx->cb = nxt_perl_psgi_cb; + PL_exit_flags |= PERL_EXIT_DESTRUCT_END; PL_origalen = 1; status = perl_run(my_perl); if (nxt_slow_path(status != 0)) { - nxt_alert(task, "PSGI: Failed to run Perl"); + nxt_unit_alert(NULL, "PSGI: Failed to run Perl"); goto fail; } sv_setsv(get_sv("0", 0), newSVpv(script, 0)); - run_module = nxt_perl_psgi_module_create(task, script); - + run_module = nxt_perl_psgi_module_create(script); if (nxt_slow_path(run_module == NULL)) { goto fail; } - status = nxt_perl_psgi_io_input_init(my_perl, &nxt_perl_psgi_arg_input); + pctx->arg_input.pctx = pctx; + status = nxt_perl_psgi_io_input_init(my_perl, &pctx->arg_input); if (nxt_slow_path(status != NXT_OK)) { - nxt_alert(task, "PSGI: Failed to init io.psgi.input"); + nxt_unit_alert(NULL, "PSGI: Failed to init io.psgi.input"); goto fail; } - status = nxt_perl_psgi_io_error_init(my_perl, &nxt_perl_psgi_arg_error); + pctx->arg_error.pctx = pctx; + status = nxt_perl_psgi_io_error_init(my_perl, &pctx->arg_error); if (nxt_slow_path(status != NXT_OK)) { - nxt_alert(task, "PSGI: Failed to init io.psgi.errors"); + nxt_unit_alert(NULL, "PSGI: Failed to init io.psgi.errors"); goto fail; } - *app = eval_pv((const char *) run_module, FALSE); + pctx->app = eval_pv(run_module, FALSE); if (SvTRUE(ERRSV)) { - nxt_alert(task, "PSGI: Failed to parse script: %s\n%s", - script, SvPV_nolen(ERRSV)); + nxt_unit_alert(NULL, "PSGI: Failed to parse script: %s\n%s", + script, SvPV_nolen(ERRSV)); goto fail; } - nxt_free(run_module); + nxt_unit_free(NULL, run_module); - return my_perl; + return NXT_UNIT_OK; fail: if (run_module != NULL) { - nxt_free(run_module); + nxt_unit_free(NULL, run_module); } perl_destruct(my_perl); perl_free(my_perl); - PERL_SYS_TERM(); - return NULL; + return NXT_UNIT_ERROR; } static SV * nxt_perl_psgi_env_create(PerlInterpreter *my_perl, - nxt_unit_request_info_t *req, nxt_perl_psgi_input_t *input) + nxt_unit_request_info_t *req) { - HV *hash_env; - AV *array_version; - uint32_t i; - nxt_unit_field_t *f; - nxt_unit_request_t *r; + HV *hash_env; + AV *array_version; + uint32_t i; + nxt_unit_field_t *f; + nxt_unit_request_t *r; + nxt_perl_psgi_ctx_t *pctx; + + pctx = req->ctx->data; hash_env = newHV(); if (nxt_slow_path(hash_env == NULL)) { @@ -664,11 +673,12 @@ nxt_perl_psgi_env_create(PerlInterpreter *my_perl, : newSVpv("http", 4))); RC(nxt_perl_psgi_add_value(my_perl, hash_env, NL("psgi.input"), - SvREFCNT_inc(nxt_perl_psgi_arg_input.io))); + SvREFCNT_inc(pctx->arg_input.io))); RC(nxt_perl_psgi_add_value(my_perl, hash_env, NL("psgi.errors"), - SvREFCNT_inc(nxt_perl_psgi_arg_error.io))); + SvREFCNT_inc(pctx->arg_error.io))); RC(nxt_perl_psgi_add_value(my_perl, hash_env, NL("psgi.multithread"), - &PL_sv_no)); + nxt_perl_psgi_ctxs != NULL + ? &PL_sv_yes : &PL_sv_no)); RC(nxt_perl_psgi_add_value(my_perl, hash_env, NL("psgi.multiprocess"), &PL_sv_yes)); RC(nxt_perl_psgi_add_value(my_perl, hash_env, NL("psgi.run_once"), @@ -1109,13 +1119,17 @@ static void nxt_perl_psgi_result_cb(PerlInterpreter *my_perl, SV *result, nxt_unit_request_info_t *req) { + nxt_perl_psgi_ctx_t *pctx; + dSP; + pctx = req->ctx->data; + ENTER; SAVETMPS; PUSHMARK(sp); - XPUSHs(newRV_noinc((SV*) nxt_perl_psgi_cb)); + XPUSHs(newRV_noinc((SV*) pctx->cb)); PUTBACK; call_sv(result, G_EVAL|G_SCALAR); @@ -1126,7 +1140,7 @@ nxt_perl_psgi_result_cb(PerlInterpreter *my_perl, SV *result, nxt_unit_error(NULL, "PSGI: Failed to execute result callback: \n%s", SvPV_nolen(ERRSV)); - nxt_perl_psgi_cb_request_done(NXT_UNIT_ERROR); + nxt_perl_psgi_cb_request_done(pctx, NXT_UNIT_ERROR); } PUTBACK; @@ -1138,90 +1152,116 @@ nxt_perl_psgi_result_cb(PerlInterpreter *my_perl, SV *result, static nxt_int_t nxt_perl_psgi_start(nxt_task_t *task, nxt_process_data_t *data) { - int rc; - nxt_unit_ctx_t *unit_ctx; - nxt_unit_init_t perl_init; - PerlInterpreter *my_perl; - nxt_common_app_conf_t *conf; - nxt_perl_psgi_module_t module; + int rc, pargc; + char **pargv, **penv; + nxt_unit_ctx_t *unit_ctx; + nxt_unit_init_t perl_init; + nxt_perl_psgi_ctx_t pctx; + nxt_perl_app_conf_t *c; + nxt_common_app_conf_t *common_conf; + + common_conf = data->app; + c = &common_conf->u.perl; + + pargc = 0; + pargv = NULL; + penv = NULL; - conf = data->app; + PERL_SYS_INIT3(&pargc, &pargv, &penv); - my_perl = nxt_perl_psgi_interpreter_init(task, conf->u.perl.script, - &module.app); + memset(&pctx, 0, sizeof(nxt_perl_psgi_ctx_t)); - if (nxt_slow_path(my_perl == NULL)) { - return NXT_ERROR; + rc = nxt_perl_psgi_ctx_init(c->script, &pctx); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + goto fail; } - module.my_perl = my_perl; - nxt_perl_psgi = my_perl; + rc = nxt_perl_psgi_init_threads(c); + + PERL_SET_CONTEXT(pctx.my_perl); + + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + goto fail; + } nxt_unit_default_init(task, &perl_init); perl_init.callbacks.request_handler = nxt_perl_psgi_request_handler; - perl_init.data = &module; - perl_init.shm_limit = conf->shm_limit; + perl_init.callbacks.ready_handler = nxt_perl_psgi_ready_handler; + perl_init.data = c; + perl_init.ctx_data = &pctx; + perl_init.shm_limit = common_conf->shm_limit; unit_ctx = nxt_unit_init(&perl_init); if (nxt_slow_path(unit_ctx == NULL)) { - return NXT_ERROR; + goto fail; } rc = nxt_unit_run(unit_ctx); + nxt_perl_psgi_join_threads(unit_ctx, c); + nxt_unit_done(unit_ctx); - nxt_perl_psgi_atexit(); + nxt_perl_psgi_ctx_free(&pctx); + + PERL_SYS_TERM(); exit(rc); return NXT_OK; + +fail: + + nxt_perl_psgi_join_threads(NULL, c); + + nxt_perl_psgi_ctx_free(&pctx); + + PERL_SYS_TERM(); + + return NXT_ERROR; } static void nxt_perl_psgi_request_handler(nxt_unit_request_info_t *req) { - SV *env, *result; - nxt_int_t rc; - PerlInterpreter *my_perl; - nxt_perl_psgi_input_t input; - nxt_perl_psgi_module_t *module; - - module = req->unit->data; - my_perl = module->my_perl; + SV *env, *result; + nxt_int_t rc; + PerlInterpreter *my_perl; + nxt_perl_psgi_ctx_t *pctx; - input.my_perl = my_perl; - input.req = req; + pctx = req->ctx->data; + my_perl = pctx->my_perl; - nxt_perl_psgi_request = req; + pctx->req = req; /* * Create environ variable for perl sub "application". * > sub application { * > my ($environ) = @_; */ - env = nxt_perl_psgi_env_create(my_perl, req, &input); + env = nxt_perl_psgi_env_create(my_perl, req); if (nxt_slow_path(env == NULL)) { nxt_unit_req_error(req, "PSGI: Failed to create 'env' for Perl Application"); nxt_unit_request_done(req, NXT_UNIT_ERROR); + pctx->req = NULL; return; } - nxt_perl_psgi_arg_input.ctx = &input; - nxt_perl_psgi_arg_error.ctx = &input; - /* Call perl sub and get result as SV*. */ - result = nxt_perl_psgi_call_var_application(my_perl, env, module->app, req); + result = nxt_perl_psgi_call_var_application(my_perl, env, pctx->app, + req); if (nxt_fast_path(SvOK(result) != 0 && SvROK(result) != 0)) { if (SvTYPE(SvRV(result)) == SVt_PVAV) { rc = nxt_perl_psgi_result_array(my_perl, result, req); nxt_unit_request_done(req, rc); + pctx->req = NULL; + goto release; } @@ -1235,6 +1275,7 @@ nxt_perl_psgi_request_handler(nxt_unit_request_info_t *req) "from Perl Application"); nxt_unit_request_done(req, NXT_UNIT_ERROR); + pctx->req = NULL; release: @@ -1243,18 +1284,181 @@ release: } +static int +nxt_perl_psgi_ready_handler(nxt_unit_ctx_t *ctx) +{ + int res; + uint32_t i; + nxt_perl_app_conf_t *c; + nxt_perl_psgi_ctx_t *pctx; + + /* Worker thread context. */ + if (!nxt_unit_is_main_ctx(ctx)) { + return NXT_UNIT_OK; + } + + c = ctx->unit->data; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + for (i = 0; i < c->threads - 1; i++) { + pctx = &nxt_perl_psgi_ctxs[i]; + + pctx->ctx = ctx; + + res = pthread_create(&pctx->thread, nxt_perl_psgi_thread_attr, + nxt_perl_psgi_thread_func, pctx); + + if (nxt_fast_path(res == 0)) { + nxt_unit_debug(ctx, "thread #%d created", (int) (i + 1)); + + } else { + nxt_unit_alert(ctx, "thread #%d create failed: %s (%d)", + (int) (i + 1), strerror(res), res); + + return NXT_UNIT_ERROR; + } + } + + return NXT_UNIT_OK; +} + + +static void * +nxt_perl_psgi_thread_func(void *data) +{ + nxt_unit_ctx_t *ctx; + nxt_perl_psgi_ctx_t *pctx; + + pctx = data; + + nxt_unit_debug(pctx->ctx, "worker thread start"); + + ctx = nxt_unit_ctx_alloc(pctx->ctx, pctx); + if (nxt_slow_path(ctx == NULL)) { + return NULL; + } + + pctx->ctx = ctx; + + PERL_SET_CONTEXT(pctx->my_perl); + + (void) nxt_unit_run(ctx); + + nxt_unit_done(ctx); + + nxt_unit_debug(NULL, "worker thread end"); + + return NULL; +} + + +static int +nxt_perl_psgi_init_threads(nxt_perl_app_conf_t *c) +{ + int rc; + uint32_t i; + static pthread_attr_t attr; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + if (c->thread_stack_size > 0) { + rc = pthread_attr_init(&attr); + if (nxt_slow_path(rc != 0)) { + nxt_unit_alert(NULL, "thread attr init failed: %s (%d)", + strerror(rc), rc); + + return NXT_UNIT_ERROR; + } + + rc = pthread_attr_setstacksize(&attr, c->thread_stack_size); + if (nxt_slow_path(rc != 0)) { + nxt_unit_alert(NULL, "thread attr set stack size failed: %s (%d)", + strerror(rc), rc); + + return NXT_UNIT_ERROR; + } + + nxt_perl_psgi_thread_attr = &attr; + } + + nxt_perl_psgi_ctxs = nxt_unit_malloc(NULL, sizeof(nxt_perl_psgi_ctx_t) + * (c->threads - 1)); + if (nxt_slow_path(nxt_perl_psgi_ctxs == NULL)) { + return NXT_UNIT_ERROR; + } + + memset(nxt_perl_psgi_ctxs, 0, sizeof(nxt_perl_psgi_ctx_t) + * (c->threads - 1)); + + for (i = 0; i < c->threads - 1; i++) { + rc = nxt_perl_psgi_ctx_init(c->script, &nxt_perl_psgi_ctxs[i]); + + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + return NXT_UNIT_ERROR; + } + } + + return NXT_UNIT_OK; +} + + static void -nxt_perl_psgi_atexit(void) +nxt_perl_psgi_join_threads(nxt_unit_ctx_t *ctx, nxt_perl_app_conf_t *c) { - dTHXa(nxt_perl_psgi); + int res; + uint32_t i; + nxt_perl_psgi_ctx_t *pctx; - nxt_perl_psgi_layer_stream_io_destroy(aTHX_ nxt_perl_psgi_arg_input.io); - nxt_perl_psgi_layer_stream_fp_destroy(aTHX_ nxt_perl_psgi_arg_input.fp); + if (nxt_perl_psgi_ctxs == NULL) { + return; + } - nxt_perl_psgi_layer_stream_io_destroy(aTHX_ nxt_perl_psgi_arg_error.io); - nxt_perl_psgi_layer_stream_fp_destroy(aTHX_ nxt_perl_psgi_arg_error.fp); + for (i = 0; i < c->threads - 1; i++) { + pctx = &nxt_perl_psgi_ctxs[i]; - perl_destruct(nxt_perl_psgi); - perl_free(nxt_perl_psgi); - PERL_SYS_TERM(); + res = pthread_join(pctx->thread, NULL); + + if (nxt_fast_path(res == 0)) { + nxt_unit_debug(ctx, "thread #%d joined", (int) (i + 1)); + + } else { + nxt_unit_alert(ctx, "thread #%d join failed: %s (%d)", + (int) (i + 1), strerror(res), res); + } + } + + for (i = 0; i < c->threads - 1; i++) { + nxt_perl_psgi_ctx_free(&nxt_perl_psgi_ctxs[i]); + } + + nxt_unit_free(NULL, nxt_perl_psgi_ctxs); +} + + +static void +nxt_perl_psgi_ctx_free(nxt_perl_psgi_ctx_t *pctx) +{ + PerlInterpreter *my_perl; + + my_perl = pctx->my_perl; + + if (nxt_slow_path(my_perl == NULL)) { + return; + } + + PERL_SET_CONTEXT(my_perl); + + nxt_perl_psgi_layer_stream_io_destroy(aTHX_ pctx->arg_input.io); + nxt_perl_psgi_layer_stream_fp_destroy(aTHX_ pctx->arg_input.fp); + + nxt_perl_psgi_layer_stream_io_destroy(aTHX_ pctx->arg_error.io); + nxt_perl_psgi_layer_stream_fp_destroy(aTHX_ pctx->arg_error.fp); + + perl_destruct(my_perl); + perl_free(my_perl); } diff --git a/src/perl/nxt_perl_psgi_layer.h b/src/perl/nxt_perl_psgi_layer.h index 3fa349c0..af18ad0d 100644 --- a/src/perl/nxt_perl_psgi_layer.h +++ b/src/perl/nxt_perl_psgi_layer.h @@ -14,7 +14,7 @@ #include -typedef struct nxt_perl_psgi_io_arg nxt_perl_psgi_io_arg_t; +typedef struct nxt_perl_psgi_io_arg_s nxt_perl_psgi_io_arg_t; typedef long (*nxt_perl_psgi_io_read_f)(PerlInterpreter *my_perl, nxt_perl_psgi_io_arg_t *arg, void *vbuf, size_t length); @@ -24,7 +24,7 @@ typedef long (*nxt_perl_psgi_io_arg_f)(PerlInterpreter *my_perl, nxt_perl_psgi_io_arg_t *arg); -struct nxt_perl_psgi_io_arg { +struct nxt_perl_psgi_io_arg_s { SV *io; PerlIO *fp; @@ -32,7 +32,7 @@ struct nxt_perl_psgi_io_arg { nxt_perl_psgi_io_read_f read; nxt_perl_psgi_io_write_f write; - void *ctx; + void *pctx; }; -- cgit From fc9a012ceb83edc5511dccb55a90ea15ab52f337 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 5 Nov 2020 16:11:01 +0300 Subject: Tests: added Perl threading tests. --- test/perl/threads/psgi.pl | 11 +++++++++++ test/test_perl_application.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 test/perl/threads/psgi.pl diff --git a/test/perl/threads/psgi.pl b/test/perl/threads/psgi.pl new file mode 100644 index 00000000..dce28f7d --- /dev/null +++ b/test/perl/threads/psgi.pl @@ -0,0 +1,11 @@ +my $app = sub { + my ($environ) = @_; + + sleep int($environ->{'HTTP_X_DELAY'}); + + return ['200', [ + 'Content-Length' => 0, + 'Psgi-Multithread' => $environ->{'psgi.multithread'}, + 'X-Thread' => $environ->{'psgi.input'} + ], []]; +}; diff --git a/test/test_perl_application.py b/test/test_perl_application.py index acd76626..78f2dd90 100644 --- a/test/test_perl_application.py +++ b/test/test_perl_application.py @@ -238,3 +238,44 @@ class TestPerlApplication(TestApplicationPerl): assert resp['status'] == 200, 'status' assert resp['body'] == 'Hello World!', 'body' + + def test_perl_application_threads(self): + self.load('threads') + + assert 'success' in self.conf( + '4', 'applications/threads/threads' + ), 'configure 4 threads' + + socks = [] + + for i in range(4): + (_, sock) = self.get( + headers={ + 'Host': 'localhost', + 'X-Delay': '2', + 'Connection': 'close', + }, + no_recv=True, + start=True, + ) + + socks.append(sock) + + threads = set() + + for sock in socks: + resp = self.recvall(sock).decode('utf-8') + + self.log_in(resp) + + resp = self._resp_to_dict(resp) + + assert resp['status'] == 200, 'status' + + threads.add(resp['headers']['X-Thread']) + + assert resp['headers']['Psgi-Multithread'] == '1', 'multithread' + + sock.close() + + assert len(socks) == len(threads), 'threads differs' -- cgit From 2220b8258f116d12f9cf6eb4133f41ec374b75f5 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 5 Nov 2020 17:02:55 +0300 Subject: Ruby: error checking during thread creation. Application terminates in case of thread creation failure. --- src/ruby/nxt_ruby.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ruby/nxt_ruby.c b/src/ruby/nxt_ruby.c index ffc1469a..698d4a43 100644 --- a/src/ruby/nxt_ruby.c +++ b/src/ruby/nxt_ruby.c @@ -1150,6 +1150,8 @@ nxt_ruby_ready_handler(nxt_unit_ctx_t *ctx) } else { nxt_unit_alert(ctx, "thread #%d create failed", (int) (i + 1)); + + return NXT_UNIT_ERROR; } } @@ -1253,7 +1255,7 @@ nxt_ruby_join_threads(nxt_unit_ctx_t *ctx, nxt_ruby_app_conf_t *c) return; } - for(i = 0; i < c->threads - 1; i++) { + for (i = 0; i < c->threads - 1; i++) { rctx = &nxt_ruby_ctxs[i]; if (rctx->thread != Qnil) { @@ -1264,8 +1266,10 @@ nxt_ruby_join_threads(nxt_unit_ctx_t *ctx, nxt_ruby_app_conf_t *c) } else { nxt_unit_debug(ctx, "thread #%d not started", (int) (i + 1)); } + } - nxt_ruby_ctx_done(rctx); + for (i = 0; i < c->threads - 1; i++) { + nxt_ruby_ctx_done(&nxt_ruby_ctxs[i]); } nxt_unit_free(ctx, nxt_ruby_ctxs); -- cgit From 78599f0d3f71953914880138fcfcdd929951fd07 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Fri, 6 Nov 2020 16:45:03 +0300 Subject: Tests: fixing racing condition in ASGI threads test. ASGI threads read all the requests from the queue before start processing it. This why test need to wait a little to let the ASGI thread start request processing and block. In virtual environment any thread or process may be delayed and only method to avoid racing is a reasonable sleep increase. --- test/test_asgi_application.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_asgi_application.py b/test/test_asgi_application.py index 179a69ee..b382e2f2 100644 --- a/test/test_asgi_application.py +++ b/test/test_asgi_application.py @@ -406,16 +406,16 @@ Connection: close self.load('threads') assert 'success' in self.conf( - '4', 'applications/threads/threads' - ), 'configure 4 threads' + '2', 'applications/threads/threads' + ), 'configure 2 threads' socks = [] - for i in range(4): + for i in range(2): (_, sock) = self.get( headers={ 'Host': 'localhost', - 'X-Delay': '2', + 'X-Delay': '3', 'Connection': 'close', }, no_recv=True, @@ -424,7 +424,7 @@ Connection: close socks.append(sock) - time.sleep(0.25) # required to avoid greedy request reading + time.sleep(1.0) # required to avoid greedy request reading threads = set() -- cgit From 702e7bcc894adaac5f409a733eaf6fd2ab822f47 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Fri, 6 Nov 2020 20:41:56 +0300 Subject: Java: fixing ClassGraph deprecated API call. The issue (deprecated API warning) introduced by ClassGraph upgrade in ccd5c695b739 commit. --- src/java/nginx/unit/Context.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/nginx/unit/Context.java b/src/java/nginx/unit/Context.java index 5aa3a982..0197858b 100644 --- a/src/java/nginx/unit/Context.java +++ b/src/java/nginx/unit/Context.java @@ -443,7 +443,7 @@ public class Context implements ServletContext, InitParams .enableClassInfo() .enableAnnotationInfo() //.enableSystemPackages() - .whitelistModules("javax.*") + .acceptModules("javax.*") //.enableAllInfo() ; -- cgit From b2771702fb6b9c7641b27f4fa1d68b9992dd5b28 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Mon, 9 Nov 2020 16:12:59 +0300 Subject: Packages: added Ubuntu 20.10 "groovy" support. --- pkg/deb/Makefile | 15 ++++++ pkg/deb/Makefile.jsc-common | 2 +- pkg/deb/Makefile.jsc13 | 71 +++++++++++++++++++++++++ pkg/deb/Makefile.jsc14 | 71 +++++++++++++++++++++++++ pkg/deb/Makefile.jsc15 | 71 +++++++++++++++++++++++++ pkg/deb/debian.module/unit.example-jsc13-config | 15 ++++++ pkg/deb/debian.module/unit.example-jsc14-config | 15 ++++++ pkg/deb/debian.module/unit.example-jsc15-config | 15 ++++++ 8 files changed, 274 insertions(+), 1 deletion(-) create mode 100644 pkg/deb/Makefile.jsc13 create mode 100644 pkg/deb/Makefile.jsc14 create mode 100644 pkg/deb/Makefile.jsc15 create mode 100644 pkg/deb/debian.module/unit.example-jsc13-config create mode 100644 pkg/deb/debian.module/unit.example-jsc14-config create mode 100644 pkg/deb/debian.module/unit.example-jsc15-config diff --git a/pkg/deb/Makefile b/pkg/deb/Makefile index efeb642f..8a02105c 100644 --- a/pkg/deb/Makefile +++ b/pkg/deb/Makefile @@ -16,6 +16,21 @@ BUILD_DEPENDS = $(BUILD_DEPENDS_unit) MODULES= +# Ubuntu 20.10 +ifeq ($(CODENAME),groovy) +include Makefile.php +include Makefile.python27 +include Makefile.python38 +include Makefile.go +include Makefile.perl +include Makefile.ruby +include Makefile.jsc-common +include Makefile.jsc11 +include Makefile.jsc13 +include Makefile.jsc14 +include Makefile.jsc15 +endif + # Ubuntu 20.04 ifeq ($(CODENAME),focal) include Makefile.php diff --git a/pkg/deb/Makefile.jsc-common b/pkg/deb/Makefile.jsc-common index 928376c3..f7a6010b 100644 --- a/pkg/deb/Makefile.jsc-common +++ b/pkg/deb/Makefile.jsc-common @@ -6,7 +6,7 @@ MODULE_SUMMARY_jsc_common= Java shared packages for NGINX Unit MODULE_VERSION_jsc_common= $(VERSION) MODULE_RELEASE_jsc_common= 1 -ifneq (,$(findstring $(CODENAME),focal eoan disco buster)) +ifneq (,$(findstring $(CODENAME),groovy focal eoan disco buster)) JAVA_MINVERSION= 11 else JAVA_MINVERSION= 8 diff --git a/pkg/deb/Makefile.jsc13 b/pkg/deb/Makefile.jsc13 new file mode 100644 index 00000000..d22944dc --- /dev/null +++ b/pkg/deb/Makefile.jsc13 @@ -0,0 +1,71 @@ +MODULES+= jsc13 +MODULE_SUFFIX_jsc13= jsc13 + +MODULE_SUMMARY_jsc13= Java 13 module for NGINX Unit + +MODULE_VERSION_jsc13= $(VERSION) +MODULE_RELEASE_jsc13= 1 + +MODULE_CONFARGS_jsc13= java --module=java13 --home=/usr/lib/jvm/java-13-openjdk-$$\(DEB_HOST_ARCH\) --jars=/usr/share/unit-jsc-common/ +MODULE_MAKEARGS_jsc13= java13 +MODULE_INSTARGS_jsc13= java13-install + +MODULE_SOURCES_jsc13= unit.example-jsc-app \ + unit.example-jsc13-config + +BUILD_DEPENDS_jsc13= openjdk-13-jdk-headless openjdk-13-jre-headless +BUILD_DEPENDS+= $(BUILD_DEPENDS_jsc13) + +MODULE_BUILD_DEPENDS_jsc13=,openjdk-13-jdk-headless +MODULE_DEPENDS_jsc13=,openjdk-13-jre-headless,unit-jsc-common (= $(MODULE_VERSION_jsc_common)-$(MODULE_RELEASE_jsc_common)~$(CODENAME)) + +define MODULE_PREINSTALL_jsc13 + mkdir -p debian/unit-jsc13/usr/share/doc/unit-jsc13/examples/jsc-app + install -m 644 -p debian/unit.example-jsc-app debian/unit-jsc13/usr/share/doc/unit-jsc13/examples/jsc-app/index.jsp + install -m 644 -p debian/unit.example-jsc13-config debian/unit-jsc13/usr/share/doc/unit-jsc13/examples/unit.config + install -m 644 -p src/java/README.JSR-340 debian/unit-jsc13/usr/share/doc/unit-jsc13/ +endef +export MODULE_PREINSTALL_jsc13 + +define MODULE_POSTINSTALL_jsc13 + cd $$\(BUILDDIR_unit\) \&\& \ + DESTDIR=$$\(INSTALLDIR\) make java-shared-uninstall +endef +export MODULE_POSTINSTALL_jsc13 + +define MODULE_POST_jsc13 +cat < Date: Tue, 10 Nov 2020 22:27:08 +0300 Subject: Tests: supporting instant app parameters in load(). --- test/test_asgi_application.py | 50 +++++++++----------------------- test/test_python_application.py | 54 +++++++---------------------------- test/unit/applications/lang/python.py | 22 ++++++++------ 3 files changed, 37 insertions(+), 89 deletions(-) diff --git a/test/test_asgi_application.py b/test/test_asgi_application.py index b382e2f2..2e73e529 100644 --- a/test/test_asgi_application.py +++ b/test/test_asgi_application.py @@ -137,23 +137,17 @@ custom-header: BLAH ), '204 header transfer encoding' def test_asgi_application_shm_ack_handle(self): - self.load('mirror') - # Minimum possible limit shm_limit = 10 * 1024 * 1024 - assert ( - 'success' in self.conf('{"shm": ' + str(shm_limit) + '}', - 'applications/mirror/limits') - ) + self.load('mirror', limits={"shm": shm_limit}) # Should exceed shm_limit max_body_size = 12 * 1024 * 1024 - assert ( - 'success' in self.conf('{"http":{"max_body_size": ' - + str(max_body_size) + ' }}', - 'settings') + assert 'success' in self.conf( + '{"http":{"max_body_size": ' + str(max_body_size) + ' }}', + 'settings' ) assert self.get()['status'] == 200, 'init' @@ -204,11 +198,6 @@ custom-header: BLAH assert resp['body'] == body, 'keep-alive 2' def test_asgi_keepalive_reconfigure(self): - skip_alert( - r'pthread_mutex.+failed', - r'failed to apply', - r'process \d+ exited on signal', - ) self.load('mirror') assert self.get()['status'] == 200, 'init' @@ -230,9 +219,8 @@ custom-header: BLAH ) assert resp['body'] == body, 'keep-alive open' - assert 'success' in self.conf( - str(i + 1), 'applications/mirror/processes' - ), 'reconfigure' + + self.load('mirror', processes=i + 1) socks.append(sock) @@ -250,9 +238,8 @@ custom-header: BLAH ) assert resp['body'] == body, 'keep-alive request' - assert 'success' in self.conf( - str(i + 1), 'applications/mirror/processes' - ), 'reconfigure 2' + + self.load('mirror', processes=i + 1) for i in range(conns): resp = self.post( @@ -266,9 +253,8 @@ custom-header: BLAH ) assert resp['body'] == body, 'keep-alive close' - assert 'success' in self.conf( - str(i + 1), 'applications/mirror/processes' - ), 'reconfigure 3' + + self.load('mirror', processes=i + 1) def test_asgi_keepalive_reconfigure_2(self): self.load('mirror') @@ -347,11 +333,7 @@ Connection: close assert resp['status'] == 200, 'reconfigure 3' def test_asgi_process_switch(self): - self.load('delayed') - - assert 'success' in self.conf( - '2', 'applications/delayed/processes' - ), 'configure 2 processes' + self.load('delayed', processes=2) self.get( headers={ @@ -382,9 +364,7 @@ Connection: close def test_asgi_application_loading_error(self): skip_alert(r'Python failed to import module "blah"') - self.load('empty') - - assert 'success' in self.conf('"blah"', 'applications/empty/module') + self.load('empty', module="blah") assert self.get()['status'] == 503, 'loading error' @@ -403,11 +383,7 @@ Connection: close ), 'last thread finished' def test_asgi_application_threads(self): - self.load('threads') - - assert 'success' in self.conf( - '2', 'applications/threads/threads' - ), 'configure 2 threads' + self.load('threads', threads=2) socks = [] diff --git a/test/test_python_application.py b/test/test_python_application.py index 780b836a..871d8bc0 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -197,11 +197,6 @@ custom-header: BLAH assert resp['body'] == body, 'keep-alive 2' def test_python_keepalive_reconfigure(self): - skip_alert( - r'pthread_mutex.+failed', - r'failed to apply', - r'process \d+ exited on signal', - ) self.load('mirror') assert self.get()['status'] == 200, 'init' @@ -223,9 +218,8 @@ custom-header: BLAH ) assert resp['body'] == body, 'keep-alive open' - assert 'success' in self.conf( - str(i + 1), 'applications/mirror/processes' - ), 'reconfigure' + + self.load('mirror', processes=i + 1) socks.append(sock) @@ -243,9 +237,8 @@ custom-header: BLAH ) assert resp['body'] == body, 'keep-alive request' - assert 'success' in self.conf( - str(i + 1), 'applications/mirror/processes' - ), 'reconfigure 2' + + self.load('mirror', processes=i + 1) for i in range(conns): resp = self.post( @@ -259,9 +252,8 @@ custom-header: BLAH ) assert resp['body'] == body, 'keep-alive close' - assert 'success' in self.conf( - str(i + 1), 'applications/mirror/processes' - ), 'reconfigure 3' + + self.load('mirror', processes=i + 1) def test_python_keepalive_reconfigure_2(self): self.load('mirror') @@ -351,11 +343,7 @@ Connection: close assert self.wait_for_record(r'At exit called\.') is not None, 'atexit' def test_python_process_switch(self): - self.load('delayed') - - assert 'success' in self.conf( - '2', 'applications/delayed/processes' - ), 'configure 2 processes' + self.load('delayed', processes=2) self.get( headers={ @@ -541,9 +529,7 @@ last line: 987654321 def test_python_application_loading_error(self): skip_alert(r'Python failed to import module "blah"') - self.load('empty') - - assert 'success' in self.conf('"blah"', 'applications/empty/module') + self.load('empty', module="blah") assert self.get()['status'] == 503, 'loading error' @@ -811,34 +797,16 @@ last line: 987654321 assert self.get()['status'] == 204, 'default application response' - assert 'success' in self.conf( - '"app"', 'applications/callable/callable' - ) + self.load('callable', callable="app") assert self.get()['status'] == 200, 'callable response' - assert 'success' in self.conf( - '"blah"', 'applications/callable/callable' - ) + self.load('callable', callable="blah") assert self.get()['status'] not in [200, 204], 'callable response inv' - assert 'success' in self.conf( - '"app"', 'applications/callable/callable' - ) - - assert self.get()['status'] == 200, 'callable response 2' - - assert 'success' in self.conf_delete('applications/callable/callable') - - assert self.get()['status'] == 204, 'default response 2' - def test_python_application_threads(self): - self.load('threads') - - assert 'success' in self.conf( - '4', 'applications/threads/threads' - ), 'configure 4 threads' + self.load('threads', threads=4) socks = [] diff --git a/test/unit/applications/lang/python.py b/test/unit/applications/lang/python.py index 374b0f9c..20d4f257 100644 --- a/test/unit/applications/lang/python.py +++ b/test/unit/applications/lang/python.py @@ -34,20 +34,24 @@ class TestApplicationPython(TestApplicationProto): script_path = '/app/python/' + name + app = { + "type": self.get_application_type(), + "processes": kwargs.pop('processes', {"spare": 0}), + "path": script_path, + "working_directory": script_path, + "module": module, + } + + for attr in ('callable', 'home', 'limits', 'path', 'threads'): + if attr in kwargs: + app[attr] = kwargs.pop(attr) + self._load_conf( { "listeners": { "*:7080": {"pass": "applications/" + quote(name, '')} }, - "applications": { - name: { - "type": self.get_application_type(), - "processes": {"spare": 0}, - "path": script_path, - "working_directory": script_path, - "module": module, - } - }, + "applications": {name: app}, }, **kwargs ) -- cgit From 4ca9ba34081c44f5d421b171ffaf874fb341d73f Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Tue, 10 Nov 2020 22:27:08 +0300 Subject: Java: fixing isolation mounts for Alpine musl. Thanks to @wujjpp. This closes #490 PR on GitHub. --- auto/modules/java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto/modules/java b/auto/modules/java index a1f27131..60415c35 100644 --- a/auto/modules/java +++ b/auto/modules/java @@ -313,7 +313,7 @@ NXT_JAVA_LIBJVM="$NXT_JAVA_LIB_SERVER_PATH/libjvm.so" if [ "$NXT_SYSTEM" = "Darwin" ]; then NXT_JAVA_LIBC_DIR="/usr/lib" else -NXT_JAVA_LIBC_DIR=`ldd "$NXT_JAVA_LIBJVM" | grep libc.so | cut -d' ' -f3` +NXT_JAVA_LIBC_DIR=`ldd "$NXT_JAVA_LIBJVM" | grep -F libc. | cut -d' ' -f3` NXT_JAVA_LIBC_DIR=`dirname $NXT_JAVA_LIBC_DIR` fi -- cgit From 5fd2933d2e54a7b5781698a670abf89b1031db44 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Tue, 10 Nov 2020 22:27:08 +0300 Subject: Python: supporting ASGI legacy protocol. Introducing manual protocol selection for 'universal' apps and frameworks. --- src/nxt_application.h | 1 + src/nxt_conf_validation.c | 26 ++++++++ src/nxt_main_process.c | 6 ++ src/python/nxt_python.c | 12 +++- src/python/nxt_python_asgi.c | 121 +++++++++++++++++++++++++++------- src/python/nxt_python_asgi.h | 2 + src/python/nxt_python_asgi_lifespan.c | 43 +++++++++++- test/python/legacy/asgi.py | 13 ++++ test/python/legacy_force/asgi.py | 17 +++++ test/test_asgi_application.py | 26 ++++++++ test/unit/applications/lang/python.py | 3 +- 11 files changed, 242 insertions(+), 28 deletions(-) create mode 100644 test/python/legacy/asgi.py create mode 100644 test/python/legacy_force/asgi.py diff --git a/src/nxt_application.h b/src/nxt_application.h index 66c0e1f7..5632f56f 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -51,6 +51,7 @@ typedef struct { nxt_str_t path; nxt_str_t module; char *callable; + nxt_str_t protocol; uint32_t threads; uint32_t thread_stack_size; } nxt_python_app_conf_t; diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 8e6279fa..fc521016 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -95,6 +95,8 @@ static nxt_int_t nxt_conf_vldt_return(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_proxy(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_python_protocol(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_thread_stack_size(nxt_conf_validation_t *vldt, @@ -493,6 +495,10 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_python_members[] = { }, { .name = nxt_string("callable"), .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("protocol"), + .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_python_protocol, }, { .name = nxt_string("threads"), .type = NXT_CONF_VLDT_INTEGER, @@ -1360,6 +1366,26 @@ nxt_conf_vldt_proxy(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } +static nxt_int_t +nxt_conf_vldt_python_protocol(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data) +{ + nxt_str_t proto; + + static const nxt_str_t wsgi = nxt_string("wsgi"); + static const nxt_str_t asgi = nxt_string("asgi"); + + nxt_conf_get_string(value, &proto); + + if (nxt_strstr_eq(&proto, &wsgi) || nxt_strstr_eq(&proto, &asgi)) { + return NXT_OK; + } + + return nxt_conf_vldt_error(vldt, "The \"protocol\" can either be " + "\"wsgi\" or \"asgi\"."); +} + + static nxt_int_t nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 99fc7995..0cde435b 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -198,6 +198,12 @@ static nxt_conf_map_t nxt_python_app_conf[] = { offsetof(nxt_common_app_conf_t, u.python.callable), }, + { + nxt_string("protocol"), + NXT_CONF_MAP_STR, + offsetof(nxt_common_app_conf_t, u.python.protocol), + }, + { nxt_string("threads"), NXT_CONF_MAP_INT32, diff --git a/src/python/nxt_python.c b/src/python/nxt_python.c index 26a6f093..faf0c0e1 100644 --- a/src/python/nxt_python.c +++ b/src/python/nxt_python.c @@ -68,6 +68,7 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) char *nxt_py_module; size_t len; PyObject *obj, *pypath, *module; + nxt_str_t proto; const char *callable; nxt_unit_ctx_t *unit_ctx; nxt_unit_init_t python_init; @@ -82,6 +83,9 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) static const char bin_python[] = "/bin/python"; #endif + static const nxt_str_t wsgi = nxt_string("wsgi"); + static const nxt_str_t asgi = nxt_string("asgi"); + app_conf = data->app; c = &app_conf->u.python; @@ -244,7 +248,13 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) python_init.shm_limit = data->app->shm_limit; python_init.callbacks.ready_handler = nxt_python_ready_handler; - if (nxt_python_asgi_check(nxt_py_application)) { + proto = c->protocol; + + if (proto.length == 0) { + proto = nxt_python_asgi_check(nxt_py_application) ? asgi : wsgi; + } + + if (nxt_strstr_eq(&proto, &asgi)) { rc = nxt_python_asgi_init(&python_init, &nxt_py_proto); } else { diff --git a/src/python/nxt_python_asgi.c b/src/python/nxt_python_asgi.c index cab73239..e11a3b6c 100644 --- a/src/python/nxt_python_asgi.c +++ b/src/python/nxt_python_asgi.c @@ -16,6 +16,7 @@ #include +static PyObject *nxt_python_asgi_get_func(PyObject *obj); static int nxt_python_asgi_ctx_data_alloc(void **pdata); static void nxt_python_asgi_ctx_data_free(void *data); static int nxt_python_asgi_startup(void *data); @@ -42,6 +43,7 @@ static PyObject *nxt_py_asgi_port_read(PyObject *self, PyObject *args); static void nxt_python_asgi_done(void); +int nxt_py_asgi_legacy; static PyObject *nxt_py_port_read; static nxt_unit_port_t *nxt_py_shared_port; @@ -64,56 +66,78 @@ int nxt_python_asgi_check(PyObject *obj) { int res; - PyObject *call; + PyObject *func; PyCodeObject *code; - if (PyFunction_Check(obj)) { - code = (PyCodeObject *) PyFunction_GET_CODE(obj); + func = nxt_python_asgi_get_func(obj); + + if (func == NULL) { + return 0; + } + + code = (PyCodeObject *) PyFunction_GET_CODE(func); + + nxt_unit_debug(NULL, "asgi_check: callable is %sa coroutine function with " + "%d argument(s)", + (code->co_flags & CO_COROUTINE) != 0 ? "" : "not ", + code->co_argcount); + + res = (code->co_flags & CO_COROUTINE) != 0 || code->co_argcount == 1; + + Py_DECREF(func); + + return res; +} + + +static PyObject * +nxt_python_asgi_get_func(PyObject *obj) +{ + PyObject *call; - return (code->co_flags & CO_COROUTINE) != 0; + if (PyFunction_Check(obj)) { + Py_INCREF(obj); + return obj; } if (PyMethod_Check(obj)) { obj = PyMethod_GET_FUNCTION(obj); - code = (PyCodeObject *) PyFunction_GET_CODE(obj); - - return (code->co_flags & CO_COROUTINE) != 0; + Py_INCREF(obj); + return obj; } call = PyObject_GetAttrString(obj, "__call__"); if (call == NULL) { - return 0; + return NULL; } if (PyFunction_Check(call)) { - code = (PyCodeObject *) PyFunction_GET_CODE(call); - - res = (code->co_flags & CO_COROUTINE) != 0; - - } else { - if (PyMethod_Check(call)) { - obj = PyMethod_GET_FUNCTION(call); + return call; + } - code = (PyCodeObject *) PyFunction_GET_CODE(obj); + if (PyMethod_Check(call)) { + obj = PyMethod_GET_FUNCTION(call); - res = (code->co_flags & CO_COROUTINE) != 0; + Py_INCREF(obj); + Py_DECREF(call); - } else { - res = 0; - } + return obj; } Py_DECREF(call); - return res; + return NULL; } int nxt_python_asgi_init(nxt_unit_init_t *init, nxt_python_proto_t *proto) { + PyObject *func; + PyCodeObject *code; + nxt_unit_debug(NULL, "asgi_init"); if (nxt_slow_path(nxt_py_asgi_str_init() != NXT_UNIT_OK)) { @@ -136,6 +160,22 @@ nxt_python_asgi_init(nxt_unit_init_t *init, nxt_python_proto_t *proto) return NXT_UNIT_ERROR; } + func = nxt_python_asgi_get_func(nxt_py_application); + if (nxt_slow_path(func == NULL)) { + nxt_unit_alert(NULL, "Python cannot find function for callable"); + return NXT_UNIT_ERROR; + } + + code = (PyCodeObject *) PyFunction_GET_CODE(func); + + if ((code->co_flags & CO_COROUTINE) == 0) { + nxt_unit_debug(NULL, "asgi: callable is not a coroutine function " + "switching to legacy mode"); + nxt_py_asgi_legacy = 1; + } + + Py_DECREF(func); + init->callbacks.request_handler = nxt_py_asgi_request_handler; init->callbacks.data_handler = nxt_py_asgi_http_data_handler; init->callbacks.websocket_handler = nxt_py_asgi_websocket_handler; @@ -366,6 +406,7 @@ static void nxt_py_asgi_request_handler(nxt_unit_request_info_t *req) { PyObject *scope, *res, *task, *receive, *send, *done, *asgi; + PyObject *stage2; nxt_py_asgi_ctx_data_t *ctx_data; if (req->request->websocket_handshake) { @@ -415,8 +456,42 @@ nxt_py_asgi_request_handler(nxt_unit_request_info_t *req) req->data = asgi; - res = PyObject_CallFunctionObjArgs(nxt_py_application, - scope, receive, send, NULL); + if (!nxt_py_asgi_legacy) { + nxt_unit_req_debug(req, "Python call ASGI 3.0 application"); + + res = PyObject_CallFunctionObjArgs(nxt_py_application, + scope, receive, send, NULL); + + } else { + nxt_unit_req_debug(req, "Python call legacy application"); + + res = PyObject_CallFunctionObjArgs(nxt_py_application, scope, NULL); + + if (nxt_slow_path(res == NULL)) { + nxt_unit_req_error(req, "Python failed to call legacy app stage1"); + nxt_python_print_exception(); + nxt_unit_request_done(req, NXT_UNIT_ERROR); + + goto release_scope; + } + + if (nxt_slow_path(PyCallable_Check(res) == 0)) { + nxt_unit_req_error(req, + "Legacy ASGI application returns not a callable"); + nxt_unit_request_done(req, NXT_UNIT_ERROR); + + Py_DECREF(res); + + goto release_scope; + } + + stage2 = res; + + res = PyObject_CallFunctionObjArgs(stage2, receive, send, NULL); + + Py_DECREF(stage2); + } + if (nxt_slow_path(res == NULL)) { nxt_unit_req_error(req, "Python failed to call the application"); nxt_python_print_exception(); diff --git a/src/python/nxt_python_asgi.h b/src/python/nxt_python_asgi.h index 69d58477..c3c3e17a 100644 --- a/src/python/nxt_python_asgi.h +++ b/src/python/nxt_python_asgi.h @@ -68,4 +68,6 @@ int nxt_py_asgi_lifespan_startup(nxt_py_asgi_ctx_data_t *ctx_data); int nxt_py_asgi_lifespan_shutdown(nxt_unit_ctx_t *ctx); +extern int nxt_py_asgi_legacy; + #endif /* _NXT_PYTHON_ASGI_H_INCLUDED_ */ diff --git a/src/python/nxt_python_asgi_lifespan.c b/src/python/nxt_python_asgi_lifespan.c index 6910d2e8..506eaf4d 100644 --- a/src/python/nxt_python_asgi_lifespan.c +++ b/src/python/nxt_python_asgi_lifespan.c @@ -71,6 +71,7 @@ nxt_py_asgi_lifespan_startup(nxt_py_asgi_ctx_data_t *ctx_data) { int rc; PyObject *scope, *res, *py_task, *receive, *send, *done; + PyObject *stage2; nxt_py_asgi_lifespan_t *lifespan; if (nxt_slow_path(PyType_Ready(&nxt_py_asgi_lifespan_type) != 0)) { @@ -129,8 +130,43 @@ nxt_py_asgi_lifespan_startup(nxt_py_asgi_ctx_data_t *ctx_data) goto release_future; } - res = PyObject_CallFunctionObjArgs(nxt_py_application, - scope, receive, send, NULL); + if (!nxt_py_asgi_legacy) { + nxt_unit_req_debug(NULL, "Python call ASGI 3.0 application"); + + res = PyObject_CallFunctionObjArgs(nxt_py_application, + scope, receive, send, NULL); + + } else { + nxt_unit_req_debug(NULL, "Python call legacy application"); + + res = PyObject_CallFunctionObjArgs(nxt_py_application, scope, NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_log(NULL, NXT_UNIT_LOG_INFO, + "ASGI Lifespan processing exception"); + nxt_python_print_exception(); + + lifespan->disabled = 1; + rc = NXT_UNIT_OK; + + goto release_scope; + } + + if (nxt_slow_path(PyCallable_Check(res) == 0)) { + nxt_unit_req_error(NULL, + "Legacy ASGI application returns not a callable"); + + Py_DECREF(res); + + goto release_scope; + } + + stage2 = res; + + res = PyObject_CallFunctionObjArgs(stage2, receive, send, NULL); + + Py_DECREF(stage2); + } + if (nxt_slow_path(res == NULL)) { nxt_unit_error(NULL, "Python failed to call the application"); nxt_python_print_exception(); @@ -143,7 +179,8 @@ nxt_py_asgi_lifespan_startup(nxt_py_asgi_ctx_data_t *ctx_data) goto release_scope; } - py_task = PyObject_CallFunctionObjArgs(ctx_data->loop_create_task, res, NULL); + py_task = PyObject_CallFunctionObjArgs(ctx_data->loop_create_task, res, + NULL); if (nxt_slow_path(py_task == NULL)) { nxt_unit_alert(NULL, "Python failed to call the create_task"); nxt_python_print_exception(); diff --git a/test/python/legacy/asgi.py b/test/python/legacy/asgi.py new file mode 100644 index 00000000..f065d026 --- /dev/null +++ b/test/python/legacy/asgi.py @@ -0,0 +1,13 @@ +def application(scope): + assert scope['type'] == 'http' + + return app_http + +async def app_http(receive, send): + await send({ + 'type': 'http.response.start', + 'status': 200, + 'headers': [ + (b'content-length', b'0'), + ] + }) diff --git a/test/python/legacy_force/asgi.py b/test/python/legacy_force/asgi.py new file mode 100644 index 00000000..2e5859f2 --- /dev/null +++ b/test/python/legacy_force/asgi.py @@ -0,0 +1,17 @@ +def application(scope, receive=None, send=None): + assert scope['type'] == 'http' + + if receive == None and send == None: + return app_http + + else: + return app_http(receive, send) + +async def app_http(receive, send): + await send({ + 'type': 'http.response.start', + 'status': 200, + 'headers': [ + (b'content-length', b'0'), + ] + }) diff --git a/test/test_asgi_application.py b/test/test_asgi_application.py index 2e73e529..e90d78bc 100644 --- a/test/test_asgi_application.py +++ b/test/test_asgi_application.py @@ -418,3 +418,29 @@ Connection: close sock.close() assert len(socks) == len(threads), 'threads differs' + + def test_asgi_application_legacy(self): + self.load('legacy') + + resp = self.get( + headers={ + 'Host': 'localhost', + 'Content-Length': '0', + 'Connection': 'close', + }, + ) + + assert resp['status'] == 200, 'status' + + def test_asgi_application_legacy_force(self): + self.load('legacy_force', protocol='asgi') + + resp = self.get( + headers={ + 'Host': 'localhost', + 'Content-Length': '0', + 'Connection': 'close', + }, + ) + + assert resp['status'] == 200, 'status' diff --git a/test/unit/applications/lang/python.py b/test/unit/applications/lang/python.py index 20d4f257..792a86fa 100644 --- a/test/unit/applications/lang/python.py +++ b/test/unit/applications/lang/python.py @@ -42,7 +42,8 @@ class TestApplicationPython(TestApplicationProto): "module": module, } - for attr in ('callable', 'home', 'limits', 'path', 'threads'): + for attr in ('callable', 'home', 'limits', 'path', 'protocol', + 'threads'): if attr in kwargs: app[attr] = kwargs.pop(attr) -- cgit From 896d8e8bfb3d8649db467d92e06c789b789d3feb Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Tue, 10 Nov 2020 22:27:08 +0300 Subject: Fixing multi-buffer body send to application. Application shared queue only capable to pass one shared memory buffer. The rest buffers in chain needs to be send directly to application in response to REQ_HEADERS_AC message. The issue can be reproduced for configurations where 'body_buffer_size' is greater than memory segment size (10 Mb). Requests with body size greater than 10 Mb are just `stuck` e.g. not passed to application awaiting for more data from router. The bug was introduced in 1d84b9e4b459 (v1.19.0). --- src/nxt_router.c | 16 +++++++++++++--- src/nxt_unit.c | 10 +++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index fbc9a4c8..95c8c160 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -3912,6 +3912,7 @@ nxt_router_req_headers_ack_handler(nxt_task_t *task, { int res; nxt_app_t *app; + nxt_buf_t *b; nxt_bool_t start_process, unlinked; nxt_port_t *app_port, *main_app_port, *idle_port; nxt_queue_link_t *idle_lnk; @@ -4009,16 +4010,25 @@ nxt_router_req_headers_ack_handler(nxt_task_t *task, req_rpc_data->app_port = app_port; - if (req_rpc_data->msg_info.body_fd != -1) { + b = req_rpc_data->msg_info.buf; + + if (b != NULL) { + /* First buffer is already sent. Start from second. */ + b = b->next; + } + + if (req_rpc_data->msg_info.body_fd != -1 || b != NULL) { nxt_debug(task, "stream #%uD: send body fd %d", req_rpc_data->stream, req_rpc_data->msg_info.body_fd); - lseek(req_rpc_data->msg_info.body_fd, 0, SEEK_SET); + if (req_rpc_data->msg_info.body_fd != -1) { + lseek(req_rpc_data->msg_info.body_fd, 0, SEEK_SET); + } res = nxt_port_socket_write(task, app_port, NXT_PORT_MSG_REQ_BODY, req_rpc_data->msg_info.body_fd, req_rpc_data->stream, - task->thread->engine->port->id, NULL); + task->thread->engine->port->id, b); if (nxt_slow_path(res != NXT_OK)) { nxt_http_request_error(task, r, NXT_HTTP_INTERNAL_SERVER_ERROR); diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 23848f5b..7230552d 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -1357,8 +1357,14 @@ nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) if (recv_msg->incoming_buf != NULL) { b = nxt_container_of(req->content_buf, nxt_unit_mmap_buf_t, buf); + while (b->next != NULL) { + b = b->next; + } + /* "Move" incoming buffer list to req_impl. */ - nxt_unit_mmap_buf_insert_tail(&b->next, recv_msg->incoming_buf); + b->next = recv_msg->incoming_buf; + b->next->prev = &b->next; + recv_msg->incoming_buf = NULL; } @@ -2988,8 +2994,6 @@ nxt_unit_request_read(nxt_unit_request_info_t *req, void *dst, size_t size) buf_res = nxt_unit_buf_read(&req->content_buf, &req->content_length, dst, size); - nxt_unit_req_debug(req, "read: %d", (int) buf_res); - if (buf_res < (ssize_t) size && req->content_fd != -1) { res = read(req->content_fd, dst, size); if (nxt_slow_path(res < 0)) { -- cgit From cb28b41311479424cdf3b217a2252518da34c23d Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Wed, 11 Nov 2020 12:09:49 +0300 Subject: PHP: prevention of consuming unread request body on finalization. The php_request_shutdown() function calls sapi_deactivate() that tries to read request body into a dummy buffer. In our case it's just waste of CPU cycles. This change is also required for the following implementation of the fastcgi_finish_request() function, where the request context can be cleared by the time of finalization. --- src/nxt_php_sapi.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 234ceef8..f6539af5 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -934,6 +934,9 @@ nxt_php_dynamic_request(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) static void nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) { +#if (PHP_VERSION_ID < 50600) + void *read_post; +#endif nxt_unit_field_t *f; zend_file_handle file_handle; @@ -990,9 +993,21 @@ nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) php_execute_script(&file_handle TSRMLS_CC); + /* Prevention of consuming possible unread request body. */ +#if (PHP_VERSION_ID < 50600) + read_post = sapi_module.read_post; + sapi_module.read_post = NULL; +#else + SG(post_read) = 1; +#endif + php_request_shutdown(NULL); nxt_unit_request_done(ctx->req, NXT_UNIT_OK); + +#if (PHP_VERSION_ID < 50600) + sapi_module.read_post = read_post; +#endif } -- cgit From e30db59168eb83ba65b55df22b021085ae700605 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Wed, 11 Nov 2020 12:09:54 +0300 Subject: PHP: implementation of the fastcgi_finish_request() function. This closes #219 issue on GitHub. --- src/nxt_php_sapi.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index f6539af5..d2fbdd27 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -8,6 +8,7 @@ #include "SAPI.h" #include "php_main.h" #include "php_variables.h" +#include "ext/standard/php_standard.h" #include #include @@ -137,16 +138,38 @@ static int nxt_php_read_post(char *buffer, uint count_bytes TSRMLS_DC); #endif +#ifdef NXT_PHP7 +#if PHP_VERSION_ID < 70200 +ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_fastcgi_finish_request, 0, 0, + _IS_BOOL, NULL, 0) +#else +ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_fastcgi_finish_request, 0, 0, + _IS_BOOL, 0) +#endif +#else /* PHP5 */ +ZEND_BEGIN_ARG_INFO_EX(arginfo_fastcgi_finish_request, 0, 0, 0) +#endif +ZEND_END_ARG_INFO() + +ZEND_FUNCTION(fastcgi_finish_request); + PHP_MINIT_FUNCTION(nxt_php_ext); ZEND_NAMED_FUNCTION(nxt_php_chdir); + +static const zend_function_entry nxt_php_ext_functions[] = { + ZEND_FE(fastcgi_finish_request, arginfo_fastcgi_finish_request) + ZEND_FE_END +}; + + zif_handler nxt_php_chdir_handler; static zend_module_entry nxt_php_unit_module = { STANDARD_MODULE_HEADER, "unit", - NULL, /* function table */ + nxt_php_ext_functions, /* function table */ PHP_MINIT(nxt_php_ext), /* initialization */ NULL, /* shutdown */ NULL, /* request initialization */ @@ -186,6 +209,55 @@ ZEND_NAMED_FUNCTION(nxt_php_chdir) } +PHP_FUNCTION(fastcgi_finish_request) +{ + nxt_php_run_ctx_t *ctx; + + if (nxt_slow_path(zend_parse_parameters_none() == FAILURE)) { +#ifdef NXT_PHP8 + RETURN_THROWS(); +#else + return; +#endif + } + + ctx = SG(server_context); + + if (nxt_slow_path(ctx->req == NULL)) { + RETURN_FALSE; + } + +#ifdef NXT_PHP7 + php_output_end_all(); + php_header(); +#else +#ifdef PHP_OUTPUT_NEWAPI + php_output_end_all(TSRMLS_C); +#else + php_end_ob_buffers(1 TSRMLS_CC); +#endif + + php_header(TSRMLS_C); +#endif + + nxt_unit_request_done(ctx->req, NXT_UNIT_OK); + ctx->req = NULL; + + PG(connection_status) = PHP_CONNECTION_ABORTED; +#ifdef NXT_PHP7 + php_output_set_status(PHP_OUTPUT_DISABLED); +#else +#ifdef PHP_OUTPUT_NEWAPI + php_output_set_status(PHP_OUTPUT_DISABLED TSRMLS_CC); +#else + php_output_set_status(0 TSRMLS_CC); +#endif +#endif + + RETURN_TRUE; +} + + static sapi_module_struct nxt_php_sapi_module = { (char *) "cli-server", @@ -1003,7 +1075,9 @@ nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) php_request_shutdown(NULL); - nxt_unit_request_done(ctx->req, NXT_UNIT_OK); + if (ctx->req != NULL) { + nxt_unit_request_done(ctx->req, NXT_UNIT_OK); + } #if (PHP_VERSION_ID < 50600) sapi_module.read_post = read_post; -- cgit From a0ee50826a4810595123460ed11629f87b1995a0 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Wed, 11 Nov 2020 14:24:40 +0000 Subject: Tests: added a test for fastcgi_finish_request() function. --- test/php/fastcgi_finish_request/index.php | 11 +++++++++++ test/test_php_application.py | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 test/php/fastcgi_finish_request/index.php diff --git a/test/php/fastcgi_finish_request/index.php b/test/php/fastcgi_finish_request/index.php new file mode 100644 index 00000000..a6211303 --- /dev/null +++ b/test/php/fastcgi_finish_request/index.php @@ -0,0 +1,11 @@ + diff --git a/test/test_php_application.py b/test/test_php_application.py index f1a9e293..578de0b7 100644 --- a/test/test_php_application.py +++ b/test/test_php_application.py @@ -94,6 +94,32 @@ class TestPHPApplication(TestApplicationPHP): assert resp['status'] == 200, 'query string empty status' assert resp['headers']['Query-String'] == '', 'query string empty' + def test_php_application_fastcgi_finish_request(self, temp_dir): + self.load('fastcgi_finish_request') + + assert self.get()['body'] == '0123' + + unit_stop() + + with open(temp_dir + '/unit.log', 'r', errors='ignore') as f: + errs = re.findall(r'Error in fastcgi_finish_request', f.read()) + + assert len(errs) == 0, 'no error' + + def test_php_application_fastcgi_finish_request_2(self, temp_dir): + self.load('fastcgi_finish_request') + + resp = self.get(url='/?skip') + assert resp['status'] == 200 + assert resp['body'] == '' + + unit_stop() + + with open(temp_dir + '/unit.log', 'r', errors='ignore') as f: + errs = re.findall(r'Error in fastcgi_finish_request', f.read()) + + assert len(errs) == 0, 'no error' + def test_php_application_query_string_absent(self): self.load('query_string') -- cgit From 3278253d51666e0b83a83b10d5ff7f391cc55f3f Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Thu, 12 Nov 2020 00:07:08 +0000 Subject: Tests: added a test for "body_buffer_size" option. --- test/test_settings.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/test_settings.py b/test/test_settings.py index 30aa2bff..22830a3b 100644 --- a/test/test_settings.py +++ b/test/test_settings.py @@ -260,3 +260,41 @@ Connection: close assert 'error' in self.conf( {'http': {'max_body_size': -1}}, 'settings' ), 'settings negative value' + + def test_settings_body_buffer_size(self): + self.load('mirror') + + assert 'success' in self.conf( + { + 'http': { + 'max_body_size': 64 * 1024 * 1024, + 'body_buffer_size': 32 * 1024 * 1024, + } + }, + 'settings', + ) + + body = '0123456789abcdef' + resp = self.post(body=body) + assert bool(resp), 'response from application' + assert resp['status'] == 200, 'status' + assert resp['body'] == body, 'body' + + body = '0123456789abcdef' * 1024 * 1024 + resp = self.post(body=body, read_buffer_size=1024 * 1024) + assert bool(resp), 'response from application 2' + assert resp['status'] == 200, 'status 2' + assert resp['body'] == body, 'body 2' + + body = '0123456789abcdef' * 2 * 1024 * 1024 + resp = self.post(body=body, read_buffer_size=1024 * 1024) + assert bool(resp), 'response from application 3' + assert resp['status'] == 200, 'status 3' + assert resp['body'] == body, 'body 3' + + body = '0123456789abcdef' * 3 * 1024 * 1024 + resp = self.post(body=body, read_buffer_size=1024 * 1024) + assert bool(resp), 'response from application 4' + assert resp['status'] == 200, 'status 4' + assert resp['body'] == body, 'body 4' + -- cgit From d6829cc93b86f10a4ad747bfcc92a9cdfb2c2519 Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Thu, 12 Nov 2020 12:04:51 +0000 Subject: Tests: removed test case that reuses rootfs path. Different applications cannot reuse the same rootfs path if not using namespaces because of globally visible builtin mount points. --- test/test_python_isolation_chroot.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/test_python_isolation_chroot.py b/test/test_python_isolation_chroot.py index 134d2b8a..8018d5b9 100644 --- a/test/test_python_isolation_chroot.py +++ b/test/test_python_isolation_chroot.py @@ -15,10 +15,6 @@ class TestPythonIsolation(TestApplicationPython): 'rootfs': temp_dir, } - self.load('empty', isolation=isolation) - - assert self.get()['status'] == 200, 'python chroot' - self.load('ns_inspect', isolation=isolation) assert ( -- cgit From 3837d28f9b7c5d0840d2e4b26f4867b66838d31b Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Fri, 13 Nov 2020 10:48:32 +0000 Subject: Isolation: added option to disable tmpfs mount. Now users can disable the default tmpfs mount point in the rootfs. { "isolation": { "automount": { "tmpfs": false } } } --- src/nxt_conf_validation.c | 3 +++ src/nxt_isolation.c | 48 ++++++++++++++++++++++++++++------------------- src/nxt_process.h | 1 + 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index fc521016..69a47274 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -841,6 +841,9 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_automount_members[] = { { .name = nxt_string("language_deps"), .type = NXT_CONF_VLDT_BOOLEAN, + }, { + .name = nxt_string("tmpfs"), + .type = NXT_CONF_VLDT_BOOLEAN, }, NXT_CONF_VLDT_END diff --git a/src/nxt_isolation.c b/src/nxt_isolation.c index e0f169aa..f0ef625f 100644 --- a/src/nxt_isolation.c +++ b/src/nxt_isolation.c @@ -484,10 +484,12 @@ nxt_isolation_set_automount(nxt_task_t *task, nxt_conf_value_t *isolation, static nxt_str_t automount_name = nxt_string("automount"); static nxt_str_t langdeps_name = nxt_string("language_deps"); + static nxt_str_t tmp_name = nxt_string("tmpfs"); automount = &process->isolation.automount; automount->language_deps = 1; + automount->tmpfs = 1; conf = nxt_conf_get_object_member(isolation, &automount_name, NULL); if (conf != NULL) { @@ -495,6 +497,11 @@ nxt_isolation_set_automount(nxt_task_t *task, nxt_conf_value_t *isolation, if (value != NULL) { automount->language_deps = nxt_conf_get_boolean(value); } + + value = nxt_conf_get_object_member(conf, &tmp_name, NULL); + if (value != NULL) { + automount->tmpfs = nxt_conf_get_boolean(value); + } } return NXT_OK; @@ -576,29 +583,32 @@ nxt_isolation_set_lang_mounts(nxt_task_t *task, nxt_process_t *process, *p = '\0'; } - mnt = nxt_array_add(mounts); - if (nxt_slow_path(mnt == NULL)) { - return NXT_ERROR; - } + if (process->isolation.automount.tmpfs) { + mnt = nxt_array_add(mounts); + if (nxt_slow_path(mnt == NULL)) { + return NXT_ERROR; + } - mnt->src = (u_char *) "tmpfs"; - mnt->name = (u_char *) "tmpfs"; - mnt->type = NXT_FS_TMP; - mnt->flags = (NXT_FS_FLAGS_NOSUID | NXT_FS_FLAGS_NODEV - | NXT_FS_FLAGS_NOEXEC); - mnt->data = (u_char *) "size=1m,mode=777"; - mnt->builtin = 1; - mnt->deps = 0; + mnt->src = (u_char *) "tmpfs"; + mnt->name = (u_char *) "tmpfs"; + mnt->type = NXT_FS_TMP; + mnt->flags = (NXT_FS_FLAGS_NOSUID + | NXT_FS_FLAGS_NODEV + | NXT_FS_FLAGS_NOEXEC); + mnt->data = (u_char *) "size=1m,mode=777"; + mnt->builtin = 1; + mnt->deps = 0; + + mnt->dst = nxt_mp_nget(mp, rootfs_len + nxt_length("/tmp") + 1); + if (nxt_slow_path(mnt->dst == NULL)) { + return NXT_ERROR; + } - mnt->dst = nxt_mp_nget(mp, rootfs_len + nxt_length("/tmp") + 1); - if (nxt_slow_path(mnt->dst == NULL)) { - return NXT_ERROR; + p = nxt_cpymem(mnt->dst, rootfs, rootfs_len); + p = nxt_cpymem(p, "/tmp", 4); + *p = '\0'; } - p = nxt_cpymem(mnt->dst, rootfs, rootfs_len); - p = nxt_cpymem(p, "/tmp", 4); - *p = '\0'; - mnt = nxt_array_add(mounts); if (nxt_slow_path(mnt == NULL)) { return NXT_ERROR; diff --git a/src/nxt_process.h b/src/nxt_process.h index ddadb08f..99ba8022 100644 --- a/src/nxt_process.h +++ b/src/nxt_process.h @@ -75,6 +75,7 @@ typedef struct { typedef struct { uint8_t language_deps; /* 1-bit */ + uint8_t tmpfs; /* 1-bit */ } nxt_process_automount_t; -- cgit From 6d2b60ff3e6f905b81c6d2fc6595b39250e3a586 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Mon, 16 Nov 2020 20:36:54 +0300 Subject: Tests: making available versions unique. --- test/conftest.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 41d1bf00..9aaf7640 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -142,10 +142,9 @@ def pytest_sessionstart(session): # discover available modules from unit.log for module in re.findall(r'module: ([a-zA-Z]+) (.*) ".*"$', log, re.M): - if module[0] not in option.available['modules']: - option.available['modules'][module[0]] = [module[1]] - else: - option.available['modules'][module[0]].append(module[1]) + versions = option.available['modules'].setdefault(module[0], []) + if module[1] not in versions: + versions.append(module[1]) # discover modules from check -- cgit From 567f0a7b3049f4532524ac35cb232cbeedb868bf Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Mon, 16 Nov 2020 20:37:01 +0300 Subject: Tests: fixing tests interrupt in terminal. KeyboardInterrupt re-raised. --- test/conftest.py | 11 ++++++++++- test/test_go_isolation.py | 2 +- test/test_java_isolation_rootfs.py | 10 ++++++++-- test/test_python_application.py | 4 ++-- test/test_static.py | 3 +++ test/test_tls.py | 4 ++++ test/unit/applications/lang/go.py | 6 ++++++ test/unit/applications/lang/java.py | 8 +++++++- test/unit/check/go.py | 3 +++ test/unit/http.py | 10 ++++++++-- 10 files changed, 52 insertions(+), 9 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 9aaf7640..3edc471d 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -215,7 +215,7 @@ def run(request): # print unit.log in case of error - if request.node.rep_call.failed: + if hasattr(request.node, 'rep_call') and request.node.rep_call.failed: _print_log() # remove unit.log @@ -284,6 +284,11 @@ def unit_stop(): retcode = p.wait(15) if retcode: return 'Child process terminated with code ' + str(retcode) + + except KeyboardInterrupt: + p.kill() + raise + except: p.kill() return 'Could not terminate unit' @@ -403,6 +408,10 @@ def waitforsocket(port): sock.connect(('127.0.0.1', port)) ret = True break + + except KeyboardInterrupt: + raise + except: sock.close() time.sleep(0.1) diff --git a/test/test_go_isolation.py b/test/test_go_isolation.py index c68925b9..e3a0a210 100644 --- a/test/test_go_isolation.py +++ b/test/test_go_isolation.py @@ -36,7 +36,7 @@ class TestGoIsolation(TestApplicationGo): try: nogroup_gid = grp.getgrnam('nogroup').gr_gid nogroup = 'nogroup' - except: + except KeyError: nogroup_gid = grp.getgrnam('nobody').gr_gid nogroup = 'nobody' diff --git a/test/test_java_isolation_rootfs.py b/test/test_java_isolation_rootfs.py index 4f9e9834..02d35a62 100644 --- a/test/test_java_isolation_rootfs.py +++ b/test/test_java_isolation_rootfs.py @@ -31,8 +31,11 @@ class TestJavaIsolationRootfs(TestApplicationJava): process.communicate() + except KeyboardInterrupt: + raise + except: - pytest.fail('Cann\'t run mount process.') + pytest.fail('Can\'t run mount process.') def teardown_method(self, is_su): if not is_su: @@ -46,8 +49,11 @@ class TestJavaIsolationRootfs(TestApplicationJava): process.communicate() + except KeyboardInterrupt: + raise + except: - pytest.fail('Cann\'t run mount process.') + pytest.fail('Can\'t run mount process.') def test_java_isolation_rootfs_chroot_war(self, is_su, temp_dir): if not is_su: diff --git a/test/test_python_application.py b/test/test_python_application.py index 871d8bc0..83b0c8f4 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -730,7 +730,7 @@ last line: 987654321 try: group_id = grp.getgrnam(group).gr_gid - except: + except KeyError: group = 'nogroup' group_id = grp.getgrnam(group).gr_gid @@ -775,7 +775,7 @@ last line: 987654321 try: grp.getgrnam(group) group = True - except: + except KeyError: group = False if group: diff --git a/test/test_static.py b/test/test_static.py index de8b71cc..a65928ca 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -134,6 +134,9 @@ class TestStatic(TestApplicationProto): open(temp_dir + '/ф а', 'a').close() utf8 = True + except KeyboardInterrupt: + raise + except: utf8 = False diff --git a/test/test_tls.py b/test/test_tls.py index 7be426b1..4cf8d22c 100644 --- a/test/test_tls.py +++ b/test/test_tls.py @@ -486,6 +486,10 @@ basicConstraints = critical,CA:TRUE""" resp = self.get_ssl( headers={'Host': 'localhost', 'Connection': 'close'}, sock=sock ) + + except KeyboardInterrupt: + raise + except: resp = None diff --git a/test/unit/applications/lang/go.py b/test/unit/applications/lang/go.py index 518cd017..866dec47 100644 --- a/test/unit/applications/lang/go.py +++ b/test/unit/applications/lang/go.py @@ -34,10 +34,16 @@ class TestApplicationGo(TestApplicationProto): option.test_dir + '/go/' + script + '/' + name + '.go', ] + if option.detailed: + print("\n$ GOPATH=" + env['GOPATH'] + " " + " ".join(args)) + try: process = subprocess.Popen(args, env=env) process.communicate() + except KeyboardInterrupt: + raise + except: return None diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py index e1bd5e0c..0ff85187 100644 --- a/test/unit/applications/lang/java.py +++ b/test/unit/applications/lang/java.py @@ -71,12 +71,18 @@ class TestApplicationJava(TestApplicationProto): ] javac.extend(src) + if option.detailed: + print("\n$ " + " ".join(javac)) + try: process = subprocess.Popen(javac, stderr=subprocess.STDOUT) process.communicate() + except KeyboardInterrupt: + raise + except: - pytest.fail('Cann\'t run javac process.') + pytest.fail('Can\'t run javac process.') def load(self, script, **kwargs): self.prepare_env(script) diff --git a/test/unit/check/go.py b/test/unit/check/go.py index dd2150eb..35b0c2d5 100644 --- a/test/unit/check/go.py +++ b/test/unit/check/go.py @@ -25,5 +25,8 @@ def check_go(current_dir, temp_dir, test_dir): if process.returncode == 0: return True + except KeyboardInterrupt: + raise + except: return None diff --git a/test/unit/http.py b/test/unit/http.py index 5f073439..8d964978 100644 --- a/test/unit/http.py +++ b/test/unit/http.py @@ -187,6 +187,10 @@ class TestHTTP(TestUnit): try: part = sock.recv(buff_size) + + except KeyboardInterrupt: + raise + except: break @@ -242,7 +246,8 @@ class TestHTTP(TestUnit): try: last_size = int(chunks[-2], 16) - except: + + except ValueError: pytest.fail('Invalid zero size chunk') if last_size != 0 or chunks[-1] != b'': @@ -252,7 +257,8 @@ class TestHTTP(TestUnit): while len(chunks) >= 2: try: size = int(chunks.pop(0), 16) - except: + + except ValueError: pytest.fail('Invalid chunk size %s' % str(size)) if size == 0: -- cgit From bbc29df8fe4400e881829741c969f2fb77487423 Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Mon, 16 Nov 2020 17:22:10 +0000 Subject: Tests: tmpfs automount. --- test/go/ns_inspect/app.go | 7 +++++++ test/test_go_isolation.py | 25 ++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/test/go/ns_inspect/app.go b/test/go/ns_inspect/app.go index 4d19a796..570580e6 100644 --- a/test/go/ns_inspect/app.go +++ b/test/go/ns_inspect/app.go @@ -7,6 +7,7 @@ import ( "unit.nginx.org/go" "os" "strconv" + "io/ioutil" ) type ( @@ -26,6 +27,7 @@ type ( GID int NS NS FileExists bool + Mounts string } ) @@ -77,6 +79,11 @@ func handler(w http.ResponseWriter, r *http.Request) { out.FileExists = err == nil } + if mounts := r.Form.Get("mounts"); mounts != "" { + data, _ := ioutil.ReadFile("/proc/self/mountinfo") + out.Mounts = string(data) + } + data, err := json.Marshal(out) if err != nil { w.WriteHeader(http.StatusInternalServerError) diff --git a/test/test_go_isolation.py b/test/test_go_isolation.py index e3a0a210..8c4a6b9c 100644 --- a/test/test_go_isolation.py +++ b/test/test_go_isolation.py @@ -332,7 +332,12 @@ class TestGoIsolation(TestApplicationGo): obj = self.getjson(url='/?file=/bin/sh')['body'] assert obj['FileExists'] == False, 'file should not exists' - def test_go_isolation_rootfs_default_tmpfs(self, is_su, temp_dir): + def test_go_isolation_rootfs_automount_tmpfs(self, is_su, temp_dir): + try: + open("/proc/self/mountinfo") + except: + pytest.skip('The system lacks /proc/self/mountinfo file') + if not is_su: if not self.isolation_key('unprivileged_userns_clone'): pytest.skip('unprivileged clone is not available') @@ -357,6 +362,20 @@ class TestGoIsolation(TestApplicationGo): self.load('ns_inspect', isolation=isolation) - obj = self.getjson(url='/?file=/tmp')['body'] + obj = self.getjson(url='/?mounts=true')['body'] + + assert ( + "/ /tmp" in obj['Mounts'] and "tmpfs" in obj['Mounts'] + ), 'app has /tmp mounted on /' + + isolation['automount'] = { + 'tmpfs': False + } + + self.load('ns_inspect', isolation=isolation) + + obj = self.getjson(url='/?mounts=true')['body'] - assert obj['FileExists'] == True, 'app has /tmp' + assert ( + "/ /tmp" not in obj['Mounts'] and "tmpfs" not in obj['Mounts'] + ), 'app has no /tmp mounted' -- cgit From e7d66acda726490fb7b8da03f0d4788857918d5a Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Mon, 16 Nov 2020 17:56:12 +0000 Subject: Isolation: added option to disable "procfs" mount. Now users can disable the default procfs mount point in the rootfs. { "isolation": { "automount": { "procfs": false } } } --- src/nxt_conf_validation.c | 3 +++ src/nxt_isolation.c | 45 +++++++++++++++++++++++++++------------------ src/nxt_process.h | 1 + 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 69a47274..dca56881 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -844,6 +844,9 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_automount_members[] = { }, { .name = nxt_string("tmpfs"), .type = NXT_CONF_VLDT_BOOLEAN, + }, { + .name = nxt_string("procfs"), + .type = NXT_CONF_VLDT_BOOLEAN, }, NXT_CONF_VLDT_END diff --git a/src/nxt_isolation.c b/src/nxt_isolation.c index f0ef625f..1e6323bc 100644 --- a/src/nxt_isolation.c +++ b/src/nxt_isolation.c @@ -485,11 +485,13 @@ nxt_isolation_set_automount(nxt_task_t *task, nxt_conf_value_t *isolation, static nxt_str_t automount_name = nxt_string("automount"); static nxt_str_t langdeps_name = nxt_string("language_deps"); static nxt_str_t tmp_name = nxt_string("tmpfs"); + static nxt_str_t proc_name = nxt_string("procfs"); automount = &process->isolation.automount; automount->language_deps = 1; automount->tmpfs = 1; + automount->procfs = 1; conf = nxt_conf_get_object_member(isolation, &automount_name, NULL); if (conf != NULL) { @@ -502,6 +504,11 @@ nxt_isolation_set_automount(nxt_task_t *task, nxt_conf_value_t *isolation, if (value != NULL) { automount->tmpfs = nxt_conf_get_boolean(value); } + + value = nxt_conf_get_object_member(conf, &proc_name, NULL); + if (value != NULL) { + automount->procfs = nxt_conf_get_boolean(value); + } } return NXT_OK; @@ -609,27 +616,29 @@ nxt_isolation_set_lang_mounts(nxt_task_t *task, nxt_process_t *process, *p = '\0'; } - mnt = nxt_array_add(mounts); - if (nxt_slow_path(mnt == NULL)) { - return NXT_ERROR; - } + if (process->isolation.automount.procfs) { + mnt = nxt_array_add(mounts); + if (nxt_slow_path(mnt == NULL)) { + return NXT_ERROR; + } - mnt->name = (u_char *) "proc"; - mnt->type = NXT_FS_PROC; - mnt->src = (u_char *) "none"; - mnt->dst = nxt_mp_nget(mp, rootfs_len + nxt_length("/proc") + 1); - if (nxt_slow_path(mnt->dst == NULL)) { - return NXT_ERROR; - } + mnt->name = (u_char *) "proc"; + mnt->type = NXT_FS_PROC; + mnt->src = (u_char *) "none"; + mnt->dst = nxt_mp_nget(mp, rootfs_len + nxt_length("/proc") + 1); + if (nxt_slow_path(mnt->dst == NULL)) { + return NXT_ERROR; + } - p = nxt_cpymem(mnt->dst, rootfs, rootfs_len); - p = nxt_cpymem(p, "/proc", 5); - *p = '\0'; + p = nxt_cpymem(mnt->dst, rootfs, rootfs_len); + p = nxt_cpymem(p, "/proc", 5); + *p = '\0'; - mnt->data = (u_char *) ""; - mnt->flags = NXT_FS_FLAGS_NOEXEC | NXT_FS_FLAGS_NOSUID; - mnt->builtin = 1; - mnt->deps = 0; + mnt->data = (u_char *) ""; + mnt->flags = NXT_FS_FLAGS_NOEXEC | NXT_FS_FLAGS_NOSUID; + mnt->builtin = 1; + mnt->deps = 0; + } qsort(mounts->elts, mounts->nelts, sizeof(nxt_fs_mount_t), nxt_isolation_mount_compare); diff --git a/src/nxt_process.h b/src/nxt_process.h index 99ba8022..7afb8803 100644 --- a/src/nxt_process.h +++ b/src/nxt_process.h @@ -76,6 +76,7 @@ typedef struct { typedef struct { uint8_t language_deps; /* 1-bit */ uint8_t tmpfs; /* 1-bit */ + uint8_t procfs; /* 1-bit */ } nxt_process_automount_t; -- cgit From fb80502513bf0140c5e595714967f75ea3e1e5d3 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Tue, 17 Nov 2020 16:50:06 +0300 Subject: HTTP parser: allowed more characters in header field names. Previously, all requests that contained in header field names characters other than alphanumeric, or "-", or "_" were rejected with a 400 "Bad Request" error response. Now, the parser allows the same set of characters as specified in RFC 7230, including: "!", "#", "$", "%", "&", "'", "*", "+", ".", "^", "`", "|", and "~". Header field names that contain only these characters are considered valid. Also, there's a new option introduced: "discard_unsafe_fields". It accepts boolean value and it is set to "true" by default. When this option is "true", all header field names that contain characters in valid range, but other than alphanumeric or "-" are skipped during parsing. When the option is "false", these header fields aren't skipped. Requests with non-valid characters in header field names according to RFC 7230 are rejected regardless of "discard_unsafe_fields" setting. This closes #422 issue on GitHub. --- src/nxt_conf_validation.c | 3 ++ src/nxt_h1proto.c | 6 +++- src/nxt_http_parse.c | 67 +++++++++++++++++++++++++++--------------- src/nxt_http_parse.h | 14 +++++---- src/nxt_router.c | 7 +++++ src/nxt_router.h | 2 ++ src/test/nxt_http_parse_test.c | 38 +++++++++++++++++++----- 7 files changed, 100 insertions(+), 37 deletions(-) diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index dca56881..f4e7c358 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -273,6 +273,9 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { }, { .name = nxt_string("body_temp_path"), .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("discard_unsafe_fields"), + .type = NXT_CONF_VLDT_BOOLEAN, }, { .name = nxt_string("websocket"), .type = NXT_CONF_VLDT_OBJECT, diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index dc23d7c4..dccbe56c 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -467,6 +467,7 @@ nxt_h1p_conn_request_init(nxt_task_t *task, void *obj, void *data) nxt_int_t ret; nxt_conn_t *c; nxt_h1proto_t *h1p; + nxt_socket_conf_t *skcf; nxt_http_request_t *r; nxt_socket_conf_joint_t *joint; @@ -503,11 +504,14 @@ nxt_h1p_conn_request_init(nxt_task_t *task, void *obj, void *data) joint->count++; r->conf = joint; + skcf = joint->socket_conf; if (c->local == NULL) { - c->local = joint->socket_conf->sockaddr; + c->local = skcf->sockaddr; } + h1p->parser.discard_unsafe_fields = skcf->discard_unsafe_fields; + nxt_h1p_conn_request_header_parse(task, c, h1p); return; } diff --git a/src/nxt_http_parse.c b/src/nxt_http_parse.c index 22004cc1..338b0a90 100644 --- a/src/nxt_http_parse.c +++ b/src/nxt_http_parse.c @@ -288,11 +288,13 @@ continue_target: case NXT_HTTP_TARGET_SPACE: rp->target_end = p; goto space_after_target; - +#if 0 case NXT_HTTP_TARGET_QUOTE_MARK: rp->quoted_target = 1; goto rest_of_target; - +#else + case NXT_HTTP_TARGET_QUOTE_MARK: +#endif case NXT_HTTP_TARGET_HASH: rp->complex_target = 1; goto rest_of_target; @@ -378,7 +380,7 @@ space_after_target: } } - rp->space_in_target = 1; + //rp->space_in_target = 1; if (rest) { goto rest_of_target; @@ -397,7 +399,7 @@ space_after_target: goto space_after_target; } - rp->space_in_target = 1; + //rp->space_in_target = 1; if (rest) { goto rest_of_target; @@ -432,7 +434,12 @@ space_after_target: *pos = p + 10; } - if (rp->complex_target != 0 || rp->quoted_target != 0) { + if (rp->complex_target != 0 +#if 0 + || rp->quoted_target != 0 +#endif + ) + { rc = nxt_http_parse_complex_target(rp); if (nxt_slow_path(rc != NXT_OK)) { @@ -518,11 +525,13 @@ nxt_http_parse_field_name(nxt_http_request_parse_t *rp, u_char **pos, static const u_char normal[256] nxt_aligned(64) = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" - "\0\0\0\0\0\0\0\0\0\0\0\0\0-\0\0" "0123456789\0\0\0\0\0\0" + /* \s ! " # $ % & ' ( ) * + , . / : ; < = > ? */ + "\0\1\0\1\1\1\1\1\0\0\1\1\0" "-" "\1\0" "0123456789" "\0\0\0\0\0\0" - /* These 64 bytes should reside in one cache line. */ - "\0abcdefghijklmnopqrstuvwxyz\0\0\0\0_" - "\0abcdefghijklmnopqrstuvwxyz\0\0\0\0\0" + /* @ [ \ ] ^ _ */ + "\0" "abcdefghijklmnopqrstuvwxyz" "\0\0\0\1\1" + /* ` { | } ~ */ + "\1" "abcdefghijklmnopqrstuvwxyz" "\0\1\0\1\0" "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" @@ -538,9 +547,14 @@ nxt_http_parse_field_name(nxt_http_request_parse_t *rp, u_char **pos, \ c = normal[ch]; \ \ - if (nxt_slow_path(c == '\0')) { \ - p = &(ch); \ - goto name_end; \ + if (nxt_slow_path(c <= '\1')) { \ + if (c == '\0') { \ + p = &(ch); \ + goto name_end; \ + } \ + \ + rp->skip_field = rp->discard_unsafe_fields; \ + c = ch; \ } \ \ hash = nxt_http_field_hash_char(hash, c); @@ -777,20 +791,25 @@ nxt_http_parse_field_end(nxt_http_request_parse_t *rp, u_char **pos, *pos = p + 1; if (rp->field_name.length != 0) { - field = nxt_list_add(rp->fields); + if (rp->skip_field) { + rp->skip_field = 0; - if (nxt_slow_path(field == NULL)) { - return NXT_ERROR; - } + } else { + field = nxt_list_add(rp->fields); - field->hash = nxt_http_field_hash_end(rp->field_hash); - field->skip = 0; - field->hopbyhop = 0; + if (nxt_slow_path(field == NULL)) { + return NXT_ERROR; + } - field->name_length = rp->field_name.length; - field->value_length = rp->field_value.length; - field->name = rp->field_name.start; - field->value = rp->field_value.start; + field->hash = nxt_http_field_hash_end(rp->field_hash); + field->skip = 0; + field->hopbyhop = 0; + + field->name_length = rp->field_name.length; + field->value_length = rp->field_value.length; + field->name = rp->field_name.start; + field->value = rp->field_value.start; + } rp->field_hash = NXT_HTTP_FIELD_HASH_INIT; @@ -1023,7 +1042,7 @@ nxt_http_parse_complex_target(nxt_http_request_parse_t *rp) break; case sw_quoted: - rp->quoted_target = 1; + //rp->quoted_target = 1; if (ch >= '0' && ch <= '9') { high = (u_char) (ch - '0'); diff --git a/src/nxt_http_parse.h b/src/nxt_http_parse.h index cbfc8433..3cd9bd15 100644 --- a/src/nxt_http_parse.h +++ b/src/nxt_http_parse.h @@ -55,15 +55,19 @@ struct nxt_http_request_parse_s { uint32_t field_hash; + uint8_t skip_field; /* 1 bit */ + uint8_t discard_unsafe_fields; /* 1 bit */ + /* target with "/." */ - uint8_t complex_target; /* 1 bit */ + uint8_t complex_target; /* 1 bit */ +#if 0 /* target with "%" */ - uint8_t quoted_target; /* 1 bit */ + uint8_t quoted_target; /* 1 bit */ /* target with " " */ - uint8_t space_in_target; /* 1 bit */ - + uint8_t space_in_target; /* 1 bit */ +#endif /* Preserve encoded '/' (%2F) and '%' (%25). */ - uint8_t encoded_slashes; /* 1 bit */ + uint8_t encoded_slashes; /* 1 bit */ }; diff --git a/src/nxt_router.c b/src/nxt_router.c index 95c8c160..9dd5c30e 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1262,6 +1262,12 @@ static nxt_conf_map_t nxt_router_http_conf[] = { NXT_CONF_MAP_STR, offsetof(nxt_socket_conf_t, body_temp_path), }, + + { + nxt_string("discard_unsafe_fields"), + NXT_CONF_MAP_INT8, + offsetof(nxt_socket_conf_t, discard_unsafe_fields), + }, }; @@ -1649,6 +1655,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, skcf->header_buffer_size = 2048; skcf->large_header_buffer_size = 8192; skcf->large_header_buffers = 4; + skcf->discard_unsafe_fields = 1; skcf->body_buffer_size = 16 * 1024; skcf->max_body_size = 8 * 1024 * 1024; skcf->proxy_header_buffer_size = 64 * 1024; diff --git a/src/nxt_router.h b/src/nxt_router.h index 512f1810..5804840f 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -191,6 +191,8 @@ typedef struct { nxt_str_t body_temp_path; + uint8_t discard_unsafe_fields; /* 1 bit */ + #if (NXT_TLS) nxt_tls_conf_t *tls; #endif diff --git a/src/test/nxt_http_parse_test.c b/src/test/nxt_http_parse_test.c index 9630b21c..540309c1 100644 --- a/src/test/nxt_http_parse_test.c +++ b/src/test/nxt_http_parse_test.c @@ -23,9 +23,15 @@ typedef struct { } nxt_http_parse_test_request_line_t; +typedef struct { + nxt_int_t result; + unsigned discard_unsafe_fields:1; +} nxt_http_parse_test_fields_t; + + typedef union { void *pointer; - nxt_int_t result; + nxt_http_parse_test_fields_t fields; nxt_http_parse_test_request_line_t request_line; } nxt_http_parse_test_data_t; @@ -324,10 +330,11 @@ static nxt_http_parse_test_case_t nxt_http_test_cases[] = { { nxt_string("GET / HTTP/1.1\r\n" "X-Unknown-Header: value\r\n" - "X-Good-Header: value\r\n\r\n"), + "X-Good-Header: value\r\n" + "!#$%&'*+.^_`|~: skipped\r\n\r\n"), NXT_DONE, &nxt_http_parse_test_fields, - { .result = NXT_OK } + { .fields = { NXT_OK, 1 } } }, { nxt_string("GET / HTTP/1.1\r\n" @@ -336,7 +343,14 @@ static nxt_http_parse_test_case_t nxt_http_test_cases[] = { "X-Bad-Header: value\r\n\r\n"), NXT_DONE, &nxt_http_parse_test_fields, - { .result = NXT_ERROR } + { .fields = { NXT_ERROR, 1 } } + }, + { + nxt_string("GET / HTTP/1.1\r\n" + "!#$%&'*+.^_`|~: allowed\r\n\r\n"), + NXT_DONE, + &nxt_http_parse_test_fields, + { .fields = { NXT_ERROR, 0 } } }, }; @@ -349,6 +363,10 @@ static nxt_http_field_proc_t nxt_http_test_fields[] = { { nxt_string("X-Good-Header"), &nxt_http_test_header_return, NXT_OK }, + + { nxt_string("!#$%&'*+.^_`|~"), + &nxt_http_test_header_return, + NXT_ERROR }, }; @@ -540,6 +558,10 @@ nxt_http_parse_test(nxt_thread_t *thr) return NXT_ERROR; } + if (test->handler == &nxt_http_parse_test_fields) { + rp.discard_unsafe_fields = test->data.fields.discard_unsafe_fields; + } + rc = nxt_http_parse_test_run(&rp, &test->request); if (rc != test->result) { @@ -740,7 +762,7 @@ nxt_http_parse_test_request_line(nxt_http_request_parse_t *rp, return NXT_ERROR; } - if (rp->complex_target != test->complex_target) { + if (rp->complex_target != (test->complex_target | test->quoted_target)) { nxt_log_alert(log, "http parse test case failed:\n" " - request:\n\"%V\"\n" " - complex_target: %d (expected: %d)", @@ -748,6 +770,7 @@ nxt_http_parse_test_request_line(nxt_http_request_parse_t *rp, return NXT_ERROR; } +#if 0 if (rp->quoted_target != test->quoted_target) { nxt_log_alert(log, "http parse test case failed:\n" " - request:\n\"%V\"\n" @@ -763,6 +786,7 @@ nxt_http_parse_test_request_line(nxt_http_request_parse_t *rp, request, rp->space_in_target, test->space_in_target); return NXT_ERROR; } +#endif return NXT_OK; } @@ -776,11 +800,11 @@ nxt_http_parse_test_fields(nxt_http_request_parse_t *rp, rc = nxt_http_fields_process(rp->fields, &nxt_http_test_fields_hash, NULL); - if (rc != data->result) { + if (rc != data->fields.result) { nxt_log_alert(log, "http parse test hash failed:\n" " - request:\n\"%V\"\n" " - result: %i (expected: %i)", - request, rc, data->result); + request, rc, data->fields.result); return NXT_ERROR; } -- cgit From 8340ca0b9c7ad4109033ccb028f87cc1b73396bc Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 18 Nov 2020 22:33:53 +0300 Subject: Libunit: improving logging consistency. Debug logging depends on macros defined in nxt_auto_config.h. --- auto/make | 2 ++ src/nxt_unit.c | 4 ++++ src/nxt_unit.h | 1 + src/test/nxt_unit_app_test.c | 2 -- src/test/nxt_unit_websocket_chat.c | 8 ++++---- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/auto/make b/auto/make index 50bc6064..fcf258fa 100644 --- a/auto/make +++ b/auto/make @@ -379,6 +379,7 @@ libunit-install: $NXT_BUILD_DIR/$NXT_LIB_UNIT_STATIC src/nxt_unit_sptr.h \ src/nxt_unit_typedefs.h \ src/nxt_unit_websocket.h \ + $NXT_BUILD_DIR/nxt_auto_config.h \ $NXT_BUILD_DIR/nxt_version.h \ src/nxt_websocket_header.h \ \$(DESTDIR)$NXT_INCDIR/ @@ -393,6 +394,7 @@ libunit-uninstall: \$(DESTDIR)$NXT_INCDIR/nxt_unit_sptr.h \ \$(DESTDIR)$NXT_INCDIR/nxt_unit_typedefs.h \ \$(DESTDIR)$NXT_INCDIR/nxt_unit_websocket.h \ + \$(DESTDIR)$NXT_INCDIR/nxt_auto_config.h \ \$(DESTDIR)$NXT_INCDIR/nxt_version.h \ \$(DESTDIR)$NXT_INCDIR/nxt_websocket_header.h @rmdir -p \$(DESTDIR)$NXT_INCDIR 2>/dev/null || true diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 7230552d..564fd094 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -6527,7 +6527,9 @@ nxt_unit_malloc(nxt_unit_ctx_t *ctx, size_t size) p = malloc(size); if (nxt_fast_path(p != NULL)) { +#if (NXT_DEBUG_ALLOC) nxt_unit_debug(ctx, "malloc(%d): %p", (int) size, p); +#endif } else { nxt_unit_alert(ctx, "malloc(%d) failed: %s (%d)", @@ -6541,7 +6543,9 @@ nxt_unit_malloc(nxt_unit_ctx_t *ctx, size_t size) void nxt_unit_free(nxt_unit_ctx_t *ctx, void *p) { +#if (NXT_DEBUG_ALLOC) nxt_unit_debug(ctx, "free(%p)", p); +#endif free(p); } diff --git a/src/nxt_unit.h b/src/nxt_unit.h index cb78e862..90cba2a3 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -12,6 +12,7 @@ #include #include +#include "nxt_auto_config.h" #include "nxt_version.h" #include "nxt_unit_typedefs.h" diff --git a/src/test/nxt_unit_app_test.c b/src/test/nxt_unit_app_test.c index b6dd13d2..a5f3728c 100644 --- a/src/test/nxt_unit_app_test.c +++ b/src/test/nxt_unit_app_test.c @@ -3,8 +3,6 @@ * Copyright (C) NGINX, Inc. */ -#include -#include #include #include #include diff --git a/src/test/nxt_unit_websocket_chat.c b/src/test/nxt_unit_websocket_chat.c index 6e274722..39f8a440 100644 --- a/src/test/nxt_unit_websocket_chat.c +++ b/src/test/nxt_unit_websocket_chat.c @@ -30,7 +30,7 @@ typedef struct { static int ws_chat_root(nxt_unit_request_info_t *req); -static void ws_chat_broadcast(const void *buf, size_t size); +static void ws_chat_broadcast(const char *buf, size_t size); static const char ws_chat_index_html[]; @@ -139,18 +139,18 @@ ws_chat_root(nxt_unit_request_info_t *req) static void -ws_chat_broadcast(const void *buf, size_t size) +ws_chat_broadcast(const char *buf, size_t size) { ws_chat_request_data_t *data; nxt_unit_request_info_t *req; - nxt_unit_debug(NULL, "broadcast: %s", buf); + nxt_unit_debug(NULL, "broadcast: %*.s", (int) size, buf); nxt_queue_each(data, &ws_chat_sessions, ws_chat_request_data_t, link) { req = nxt_unit_get_request_info_from_data(data); - nxt_unit_req_debug(req, "broadcast: %s", buf); + nxt_unit_req_debug(req, "send: %*.s", (int) size, buf); nxt_unit_websocket_send(req, NXT_WEBSOCKET_OP_TEXT, 1, buf, size); } nxt_queue_loop; -- cgit From 0ec69aa46e3577ef44060b9dd8576faab4a863ce Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 18 Nov 2020 22:33:53 +0300 Subject: Libunit: fixing racing condition for port add / state change. The issue only occurred in Go applications because "port_send" is overloaded only in Go. To reproduce it, send multiple concurrent requests to the application after it has initialised. The warning message "[unit] [go] port NNN:dd not found" is the first visible aspect of the issue; the second and more valuable one is a closed connection, an error response, or a hanging response to some requests. When the application starts, it is unaware of the router's worker thread ports, so it requests the ports from the router after receiving requests from the corresponding router worker threads. When multiple requests are processed simultaneously, the router port may be required by several requests, so request processing starts only after the application receives the required port information. The port should be added to the Go port repository after its 'ready' flag is updated. Otherwise, Unit may start processing some requests and use the port before it is in the repository. The issue was introduced in changeset 78836321a126. --- src/nxt_unit.c | 126 +++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 87 insertions(+), 39 deletions(-) diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 564fd094..69cae8bb 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -147,6 +147,8 @@ nxt_inline void nxt_unit_port_use(nxt_unit_port_t *port); nxt_inline void nxt_unit_port_release(nxt_unit_port_t *port); static nxt_unit_port_t *nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *queue); +static void nxt_unit_process_awaiting_req(nxt_unit_ctx_t *ctx, + nxt_queue_t *awaiting_req); static void nxt_unit_remove_port(nxt_unit_impl_t *lib, nxt_unit_port_id_t *port_id); static nxt_unit_port_t *nxt_unit_remove_port_unsafe(nxt_unit_impl_t *lib, @@ -5340,14 +5342,12 @@ nxt_inline void nxt_unit_port_release(nxt_unit_port_t *port) static nxt_unit_port_t * nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *queue) { - int rc; - nxt_queue_t awaiting_req; - nxt_unit_impl_t *lib; - nxt_unit_port_t *old_port; - nxt_unit_process_t *process; - nxt_unit_ctx_impl_t *ctx_impl; - nxt_unit_port_impl_t *new_port, *old_port_impl; - nxt_unit_request_info_impl_t *req_impl; + int rc, ready; + nxt_queue_t awaiting_req; + nxt_unit_impl_t *lib; + nxt_unit_port_t *old_port; + nxt_unit_process_t *process; + nxt_unit_port_impl_t *new_port, *old_port_impl; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); @@ -5396,46 +5396,45 @@ nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *queue) old_port_impl->queue = queue; } - if (!nxt_queue_is_empty(&old_port_impl->awaiting_req)) { - nxt_queue_add(&awaiting_req, &old_port_impl->awaiting_req); - nxt_queue_init(&old_port_impl->awaiting_req); - } - - old_port_impl->ready = (port->in_fd != -1 || port->out_fd != -1); + ready = (port->in_fd != -1 || port->out_fd != -1); - pthread_mutex_unlock(&lib->mutex); + /* + * Port can be market as 'ready' only after callbacks.add_port() call. + * Otherwise, request may try to use the port before callback. + */ + if (lib->callbacks.add_port == NULL && ready) { + old_port_impl->ready = ready; - if (lib->callbacks.add_port != NULL - && (port->in_fd != -1 || port->out_fd != -1)) - { - lib->callbacks.add_port(ctx, old_port); + if (!nxt_queue_is_empty(&old_port_impl->awaiting_req)) { + nxt_queue_add(&awaiting_req, &old_port_impl->awaiting_req); + nxt_queue_init(&old_port_impl->awaiting_req); + } } - nxt_queue_each(req_impl, &awaiting_req, - nxt_unit_request_info_impl_t, port_wait_link) - { - nxt_queue_remove(&req_impl->port_wait_link); - - ctx_impl = nxt_container_of(req_impl->req.ctx, nxt_unit_ctx_impl_t, - ctx); + pthread_mutex_unlock(&lib->mutex); - pthread_mutex_lock(&ctx_impl->mutex); + if (lib->callbacks.add_port != NULL && ready) { + lib->callbacks.add_port(ctx, old_port); - nxt_queue_insert_tail(&ctx_impl->ready_req, - &req_impl->port_wait_link); + pthread_mutex_lock(&lib->mutex); - pthread_mutex_unlock(&ctx_impl->mutex); + old_port_impl->ready = ready; - nxt_atomic_fetch_add(&ctx_impl->wait_items, -1); + if (!nxt_queue_is_empty(&old_port_impl->awaiting_req)) { + nxt_queue_add(&awaiting_req, &old_port_impl->awaiting_req); + nxt_queue_init(&old_port_impl->awaiting_req); + } - nxt_unit_awake_ctx(ctx, ctx_impl); + pthread_mutex_unlock(&lib->mutex); + } - } nxt_queue_loop; + nxt_unit_process_awaiting_req(ctx, &awaiting_req); return old_port; } new_port = NULL; + ready = 0; nxt_unit_debug(ctx, "add_port: port{%d,%d} in_fd %d out_fd %d queue %p", port->id.pid, port->id.id, @@ -5478,13 +5477,21 @@ nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *queue) new_port->use_count = 2; new_port->process = process; - new_port->ready = (port->in_fd != -1 || port->out_fd != -1); new_port->queue = queue; new_port->from_socket = 0; new_port->socket_rbuf = NULL; nxt_queue_init(&new_port->awaiting_req); + ready = (port->in_fd != -1 || port->out_fd != -1); + + if (lib->callbacks.add_port == NULL) { + new_port->ready = ready; + + } else { + new_port->ready = 0; + } + process = NULL; unlock: @@ -5495,14 +5502,55 @@ unlock: nxt_unit_process_release(process); } - if (lib->callbacks.add_port != NULL - && new_port != NULL - && (port->in_fd != -1 || port->out_fd != -1)) - { + if (lib->callbacks.add_port != NULL && new_port != NULL && ready) { lib->callbacks.add_port(ctx, &new_port->port); + + nxt_queue_init(&awaiting_req); + + pthread_mutex_lock(&lib->mutex); + + new_port->ready = 1; + + if (!nxt_queue_is_empty(&new_port->awaiting_req)) { + nxt_queue_add(&awaiting_req, &new_port->awaiting_req); + nxt_queue_init(&new_port->awaiting_req); + } + + pthread_mutex_unlock(&lib->mutex); + + nxt_unit_process_awaiting_req(ctx, &awaiting_req); } - return &new_port->port; + return (new_port == NULL) ? NULL : &new_port->port; +} + + +static void +nxt_unit_process_awaiting_req(nxt_unit_ctx_t *ctx, nxt_queue_t *awaiting_req) +{ + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_request_info_impl_t *req_impl; + + nxt_queue_each(req_impl, awaiting_req, + nxt_unit_request_info_impl_t, port_wait_link) + { + nxt_queue_remove(&req_impl->port_wait_link); + + ctx_impl = nxt_container_of(req_impl->req.ctx, nxt_unit_ctx_impl_t, + ctx); + + pthread_mutex_lock(&ctx_impl->mutex); + + nxt_queue_insert_tail(&ctx_impl->ready_req, + &req_impl->port_wait_link); + + pthread_mutex_unlock(&ctx_impl->mutex); + + nxt_atomic_fetch_add(&ctx_impl->wait_items, -1); + + nxt_unit_awake_ctx(ctx, ctx_impl); + + } nxt_queue_loop; } -- cgit From d26afcb481d97cd71db014b16bde44e807043a2b Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 18 Nov 2020 22:33:53 +0300 Subject: Libunit: fixing racing condition in request struct recycling. The issue occurred under highly concurrent request load in Go applications. Such applications are multi-threaded but use a single libunit context; any thread-safe code in the libunit context is only required for Go applications. As a result of improper request state reset, the recycled request structure was recovered in the released state, so further operations with this request resulted in 'response already sent' warnings. However, the actual response was never delivered to the router and the client. --- src/nxt_unit.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 69cae8bb..f0c68374 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -1751,6 +1751,8 @@ nxt_unit_request_info_release(nxt_unit_request_info_t *req) req->response_port = NULL; } + req_impl->state = NXT_UNIT_RS_RELEASED; + pthread_mutex_lock(&ctx_impl->mutex); nxt_queue_remove(&req_impl->link); @@ -1758,8 +1760,6 @@ nxt_unit_request_info_release(nxt_unit_request_info_t *req) nxt_queue_insert_tail(&ctx_impl->free_req, &req_impl->link); pthread_mutex_unlock(&ctx_impl->mutex); - - req_impl->state = NXT_UNIT_RS_RELEASED; } -- cgit From 8132e1f700934a32bc9e3fb0ab66f550a335a326 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 18 Nov 2020 22:33:53 +0300 Subject: Go: removing C proxy functions and re-using goroutines. --- go/nxt_cgo_lib.c | 151 +++++------------------------------------------------ go/nxt_cgo_lib.h | 32 ++++-------- go/port.go | 48 +++++++++++------ go/request.go | 154 ++++++++++++++++++++++++++----------------------------- go/response.go | 32 ++++-------- go/unit.go | 34 +++++++----- src/nxt_unit.c | 75 ++++++++++++++++++++++----- src/nxt_unit.h | 2 + 8 files changed, 225 insertions(+), 303 deletions(-) diff --git a/go/nxt_cgo_lib.c b/go/nxt_cgo_lib.c index f7171f55..330697c1 100644 --- a/go/nxt_cgo_lib.c +++ b/go/nxt_cgo_lib.c @@ -10,16 +10,10 @@ #include -static void nxt_cgo_request_handler(nxt_unit_request_info_t *req); -static nxt_cgo_str_t *nxt_cgo_str_init(nxt_cgo_str_t *dst, - nxt_unit_sptr_t *sptr, uint32_t length); -static int nxt_cgo_add_port(nxt_unit_ctx_t *, nxt_unit_port_t *port); -static void nxt_cgo_remove_port(nxt_unit_t *, nxt_unit_port_t *port); static ssize_t nxt_cgo_port_send(nxt_unit_ctx_t *, nxt_unit_port_t *port, const void *buf, size_t buf_size, const void *oob, size_t oob_size); static ssize_t nxt_cgo_port_recv(nxt_unit_ctx_t *, nxt_unit_port_t *port, void *buf, size_t buf_size, void *oob, size_t oob_size); -static void nxt_cgo_shm_ack_handler(nxt_unit_ctx_t *ctx); int nxt_cgo_run(uintptr_t handler) @@ -30,12 +24,12 @@ nxt_cgo_run(uintptr_t handler) memset(&init, 0, sizeof(init)); - init.callbacks.request_handler = nxt_cgo_request_handler; - init.callbacks.add_port = nxt_cgo_add_port; - init.callbacks.remove_port = nxt_cgo_remove_port; + init.callbacks.request_handler = nxt_go_request_handler; + init.callbacks.add_port = nxt_go_add_port; + init.callbacks.remove_port = nxt_go_remove_port; init.callbacks.port_send = nxt_cgo_port_send; init.callbacks.port_recv = nxt_cgo_port_recv; - init.callbacks.shm_ack_handler = nxt_cgo_shm_ack_handler; + init.callbacks.shm_ack_handler = nxt_go_shm_ack_handler; init.data = (void *) handler; @@ -52,76 +46,6 @@ nxt_cgo_run(uintptr_t handler) } -static void -nxt_cgo_request_handler(nxt_unit_request_info_t *req) -{ - uint32_t i; - uintptr_t go_req; - nxt_cgo_str_t method, uri, name, value, proto, host, remote_addr; - nxt_unit_field_t *f; - nxt_unit_request_t *r; - - r = req->request; - - go_req = nxt_go_request_create((uintptr_t) req, - nxt_cgo_str_init(&method, &r->method, r->method_length), - nxt_cgo_str_init(&uri, &r->target, r->target_length)); - - nxt_go_request_set_proto(go_req, - nxt_cgo_str_init(&proto, &r->version, r->version_length), 1, 1); - - for (i = 0; i < r->fields_count; i++) { - f = &r->fields[i]; - - nxt_go_request_add_header(go_req, - nxt_cgo_str_init(&name, &f->name, f->name_length), - nxt_cgo_str_init(&value, &f->value, f->value_length)); - } - - nxt_go_request_set_content_length(go_req, r->content_length); - nxt_go_request_set_host(go_req, - nxt_cgo_str_init(&host, &r->server_name, r->server_name_length)); - nxt_go_request_set_remote_addr(go_req, - nxt_cgo_str_init(&remote_addr, &r->remote, r->remote_length)); - - if (r->tls) { - nxt_go_request_set_tls(go_req); - } - - nxt_go_request_handler(go_req, (uintptr_t) req->unit->data); -} - - -static nxt_cgo_str_t * -nxt_cgo_str_init(nxt_cgo_str_t *dst, nxt_unit_sptr_t *sptr, uint32_t length) -{ - dst->length = length; - dst->start = nxt_unit_sptr_get(sptr); - - return dst; -} - - -static int -nxt_cgo_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) -{ - nxt_go_add_port((uintptr_t) ctx, port->id.pid, port->id.id, - port->in_fd, port->out_fd); - - port->in_fd = -1; - port->out_fd = -1; - - return NXT_UNIT_OK; -} - - -static void -nxt_cgo_remove_port(nxt_unit_t *unit, nxt_unit_port_t *port) -{ - nxt_go_remove_port(port->id.pid, port->id.id); -} - - static ssize_t nxt_cgo_port_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, const void *buf, size_t buf_size, const void *oob, size_t oob_size) @@ -140,78 +64,31 @@ nxt_cgo_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, } -static void -nxt_cgo_shm_ack_handler(nxt_unit_ctx_t *ctx) -{ - return nxt_go_shm_ack_handler(); -} - - -int -nxt_cgo_response_create(uintptr_t req, int status, int fields, - uint32_t fields_size) -{ - return nxt_unit_response_init((nxt_unit_request_info_t *) req, - status, fields, fields_size); -} - - -int -nxt_cgo_response_add_field(uintptr_t req, uintptr_t name, uint8_t name_len, - uintptr_t value, uint32_t value_len) -{ - return nxt_unit_response_add_field((nxt_unit_request_info_t *) req, - (char *) name, name_len, - (char *) value, value_len); -} - - -int -nxt_cgo_response_send(uintptr_t req) -{ - return nxt_unit_response_send((nxt_unit_request_info_t *) req); -} - - ssize_t -nxt_cgo_response_write(uintptr_t req, uintptr_t start, uint32_t len) +nxt_cgo_response_write(nxt_unit_request_info_t *req, uintptr_t start, + uint32_t len) { - return nxt_unit_response_write_nb((nxt_unit_request_info_t *) req, - (void *) start, len, 0); + return nxt_unit_response_write_nb(req, (void *) start, len, 0); } ssize_t -nxt_cgo_request_read(uintptr_t req, uintptr_t dst, uint32_t dst_len) -{ - return nxt_unit_request_read((nxt_unit_request_info_t *) req, - (void *) dst, dst_len); -} - - -int -nxt_cgo_request_close(uintptr_t req) +nxt_cgo_request_read(nxt_unit_request_info_t *req, uintptr_t dst, + uint32_t dst_len) { - return 0; + return nxt_unit_request_read(req, (void *) dst, dst_len); } void -nxt_cgo_request_done(uintptr_t req, int res) +nxt_cgo_warn(const char *msg, uint32_t msg_len) { - nxt_unit_request_done((nxt_unit_request_info_t *) req, res); -} - - -void -nxt_cgo_unit_run_shared(uintptr_t ctx) -{ - nxt_unit_run_shared((nxt_unit_ctx_t *) ctx); + nxt_unit_warn(NULL, "%.*s", (int) msg_len, (char *) msg); } void -nxt_cgo_warn(uintptr_t msg, uint32_t msg_len) +nxt_cgo_alert(const char *msg, uint32_t msg_len) { - nxt_unit_warn(NULL, "%.*s", (int) msg_len, (char *) msg); + nxt_unit_alert(NULL, "%.*s", (int) msg_len, (char *) msg); } diff --git a/go/nxt_cgo_lib.h b/go/nxt_cgo_lib.h index fa515be5..3705d1ef 100644 --- a/go/nxt_cgo_lib.h +++ b/go/nxt_cgo_lib.h @@ -11,32 +11,22 @@ #include #include #include +#include +#include -typedef struct { - int length; - char *start; -} nxt_cgo_str_t; +enum { + NXT_FIELDS_OFFSET = offsetof(nxt_unit_request_t, fields) +}; int nxt_cgo_run(uintptr_t handler); -int nxt_cgo_response_create(uintptr_t req, int code, int fields, - uint32_t fields_size); +ssize_t nxt_cgo_response_write(nxt_unit_request_info_t *req, + uintptr_t src, uint32_t len); -int nxt_cgo_response_add_field(uintptr_t req, uintptr_t name, uint8_t name_len, - uintptr_t value, uint32_t value_len); +ssize_t nxt_cgo_request_read(nxt_unit_request_info_t *req, + uintptr_t dst, uint32_t dst_len); -int nxt_cgo_response_send(uintptr_t req); - -ssize_t nxt_cgo_response_write(uintptr_t req, uintptr_t src, uint32_t len); - -ssize_t nxt_cgo_request_read(uintptr_t req, uintptr_t dst, uint32_t dst_len); - -int nxt_cgo_request_close(uintptr_t req); - -void nxt_cgo_request_done(uintptr_t req, int res); - -void nxt_cgo_unit_run_shared(uintptr_t ctx); - -void nxt_cgo_warn(uintptr_t msg, uint32_t msg_len); +void nxt_cgo_warn(const char *msg, uint32_t msg_len); +void nxt_cgo_alert(const char *msg, uint32_t msg_len); #endif /* _NXT_CGO_LIB_H_INCLUDED_ */ diff --git a/go/port.go b/go/port.go index 64004d91..78351322 100644 --- a/go/port.go +++ b/go/port.go @@ -11,6 +11,7 @@ package unit import "C" import ( + "io" "net" "os" "sync" @@ -79,13 +80,13 @@ func getUnixConn(fd int) *net.UnixConn { c, err := net.FileConn(f) if err != nil { - nxt_go_warn("FileConn error %s", err) + nxt_go_alert("FileConn error %s", err) return nil } uc, ok := c.(*net.UnixConn) if !ok { - nxt_go_warn("Not a Unix-domain socket %d", fd) + nxt_go_alert("Not a Unix-domain socket %d", fd) return nil } @@ -93,30 +94,37 @@ func getUnixConn(fd int) *net.UnixConn { } //export nxt_go_add_port -func nxt_go_add_port(ctx C.uintptr_t, pid C.int, id C.int, rcv C.int, snd C.int) { - p := &port{ +func nxt_go_add_port(ctx *C.nxt_unit_ctx_t, p *C.nxt_unit_port_t) C.int { + + new_port := &port{ key: port_key{ - pid: int(pid), - id: int(id), + pid: int(p.id.pid), + id: int(p.id.id), }, - rcv: getUnixConn(int(rcv)), - snd: getUnixConn(int(snd)), + rcv: getUnixConn(int(p.in_fd)), + snd: getUnixConn(int(p.out_fd)), } - add_port(p) + add_port(new_port) + + p.in_fd = -1 + p.out_fd = -1 - if id == 65535 { - go func(ctx C.uintptr_t) { - C.nxt_cgo_unit_run_shared(ctx); + if new_port.key.id == 65535 { + go func(ctx *C.nxt_unit_ctx_t) { + C.nxt_unit_run_shared(ctx); }(ctx) } + + return C.NXT_UNIT_OK } //export nxt_go_remove_port -func nxt_go_remove_port(pid C.int, id C.int) { +func nxt_go_remove_port(unit *C.nxt_unit_t, p *C.nxt_unit_port_t) { + key := port_key{ - pid: int(pid), - id: int(id), + pid: int(p.id.pid), + id: int(p.id.id), } port_registry_.Lock() @@ -139,7 +147,7 @@ func nxt_go_port_send(pid C.int, id C.int, buf unsafe.Pointer, buf_size C.int, p := find_port(key) if p == nil { - nxt_go_warn("port %d:%d not found", pid, id) + nxt_go_alert("port %d:%d not found", pid, id) return 0 } @@ -167,7 +175,7 @@ func nxt_go_port_recv(pid C.int, id C.int, buf unsafe.Pointer, buf_size C.int, p := find_port(key) if p == nil { - nxt_go_warn("port %d:%d not found", pid, id) + nxt_go_alert("port %d:%d not found", pid, id) return 0 } @@ -175,6 +183,12 @@ func nxt_go_port_recv(pid C.int, id C.int, buf unsafe.Pointer, buf_size C.int, GoBytes(oob, oob_size)) if err != nil { + if nerr, ok := err.(*net.OpError); ok { + if nerr.Err == io.EOF { + return 0 + } + } + nxt_go_warn("read result %d (%d), %s", n, oobn, err) n = -1 diff --git a/go/request.go b/go/request.go index 1d8c6702..7e2e848a 100644 --- a/go/request.go +++ b/go/request.go @@ -19,9 +19,9 @@ import ( ) type request struct { - req http.Request - resp *response - c_req C.uintptr_t + req http.Request + resp response + c_req *C.nxt_unit_request_info_t } func (r *request) Read(p []byte) (n int, err error) { @@ -35,110 +35,102 @@ func (r *request) Read(p []byte) (n int, err error) { } func (r *request) Close() error { - C.nxt_cgo_request_close(r.c_req) return nil } -func (r *request) response() *response { - if r.resp == nil { - r.resp = new_response(r.c_req, &r.req) - } +func new_request(c_req *C.nxt_unit_request_info_t) (r *request, err error) { + req := c_req.request - return r.resp -} + uri := GoStringN(&req.target, C.int(req.target_length)) -func (r *request) done() { - resp := r.response() - if !resp.headerSent { - resp.WriteHeader(http.StatusOK) + URL, err := url.ParseRequestURI(uri) + if err != nil { + return nil, err } - C.nxt_cgo_request_done(r.c_req, 0) -} - -func get_request(go_req uintptr) *request { - return (*request)(unsafe.Pointer(go_req)) -} - -//export nxt_go_request_create -func nxt_go_request_create(c_req C.uintptr_t, - c_method *C.nxt_cgo_str_t, c_uri *C.nxt_cgo_str_t) uintptr { - - uri := C.GoStringN(c_uri.start, c_uri.length) - var URL *url.URL - var err error - if URL, err = url.ParseRequestURI(uri); err != nil { - return 0 - } + proto := GoStringN(&req.version, C.int(req.version_length)) - r := &request{ - req: http.Request{ - Method: C.GoStringN(c_method.start, c_method.length), - URL: URL, - Header: http.Header{}, - Body: nil, + r = &request{ + req: http.Request { + URL: URL, + Header: http.Header{}, RequestURI: uri, + Method: GoStringN(&req.method, C.int(req.method_length)), + Proto: proto, + ProtoMajor: 1, + ProtoMinor: int(proto[7] - '0'), + ContentLength: int64(req.content_length), + Host: GoStringN(&req.server_name, C.int(req.server_name_length)), + RemoteAddr: GoStringN(&req.remote, C.int(req.remote_length)), }, + resp: response{header: http.Header{}, c_req: c_req}, c_req: c_req, } + r.req.Body = r - return uintptr(unsafe.Pointer(r)) -} + if req.tls != 0 { + r.req.TLS = &tls.ConnectionState{ } + r.req.URL.Scheme = "https" -//export nxt_go_request_set_proto -func nxt_go_request_set_proto(go_req uintptr, proto *C.nxt_cgo_str_t, - maj C.int, min C.int) { + } else { + r.req.URL.Scheme = "http" + } - r := get_request(go_req) - r.req.Proto = C.GoStringN(proto.start, proto.length) - r.req.ProtoMajor = int(maj) - r.req.ProtoMinor = int(min) -} + fields := get_fields(req) -//export nxt_go_request_add_header -func nxt_go_request_add_header(go_req uintptr, name *C.nxt_cgo_str_t, - value *C.nxt_cgo_str_t) { + for i := 0; i < len(fields); i++ { + f := &fields[i] - r := get_request(go_req) - r.req.Header.Add(C.GoStringN(name.start, name.length), - C.GoStringN(value.start, value.length)) -} + n := GoStringN(&f.name, C.int(f.name_length)) + v := GoStringN(&f.value, C.int(f.value_length)) -//export nxt_go_request_set_content_length -func nxt_go_request_set_content_length(go_req uintptr, l C.int64_t) { - get_request(go_req).req.ContentLength = int64(l) -} + r.req.Header.Add(n, v) + } -//export nxt_go_request_set_host -func nxt_go_request_set_host(go_req uintptr, host *C.nxt_cgo_str_t) { - get_request(go_req).req.Host = C.GoStringN(host.start, host.length) + return r, nil } -//export nxt_go_request_set_url -func nxt_go_request_set_url(go_req uintptr, scheme *C.char) { - get_request(go_req).req.URL.Scheme = C.GoString(scheme) -} +func get_fields(req *C.nxt_unit_request_t) []C.nxt_unit_field_t { + f := uintptr(unsafe.Pointer(req)) + uintptr(C.NXT_FIELDS_OFFSET) -//export nxt_go_request_set_remote_addr -func nxt_go_request_set_remote_addr(go_req uintptr, addr *C.nxt_cgo_str_t) { + h := &slice_header{ + Data: unsafe.Pointer(f), + Len: int(req.fields_count), + Cap: int(req.fields_count), + } - get_request(go_req).req.RemoteAddr = C.GoStringN(addr.start, addr.length) + return *(*[]C.nxt_unit_field_t)(unsafe.Pointer(h)) } -//export nxt_go_request_set_tls -func nxt_go_request_set_tls(go_req uintptr) { +//export nxt_go_request_handler +func nxt_go_request_handler(c_req *C.nxt_unit_request_info_t) { - get_request(go_req).req.TLS = &tls.ConnectionState{ } -} + go func(c_req *C.nxt_unit_request_info_t, handler http.Handler) { -//export nxt_go_request_handler -func nxt_go_request_handler(go_req uintptr, h uintptr) { - r := get_request(go_req) - handler := get_handler(h) - - go func(r *request) { - handler.ServeHTTP(r.response(), &r.req) - r.done() - }(r) + ctx := c_req.ctx + + for { + r, err := new_request(c_req) + + if err == nil { + handler.ServeHTTP(&r.resp, &r.req) + + if !r.resp.header_sent { + r.resp.WriteHeader(http.StatusOK) + } + + C.nxt_unit_request_done(c_req, C.NXT_UNIT_OK) + + } else { + C.nxt_unit_request_done(c_req, C.NXT_UNIT_ERROR) + } + + c_req = C.nxt_unit_dequeue_request(ctx) + if c_req == nil { + break + } + } + + }(c_req, get_handler(uintptr(c_req.unit.data))) } diff --git a/go/response.go b/go/response.go index bfa79656..a1af30b3 100644 --- a/go/response.go +++ b/go/response.go @@ -16,28 +16,17 @@ import ( type response struct { header http.Header - headerSent bool - req *http.Request - c_req C.uintptr_t + header_sent bool + c_req *C.nxt_unit_request_info_t ch chan int } -func new_response(c_req C.uintptr_t, req *http.Request) *response { - resp := &response{ - header: http.Header{}, - req: req, - c_req: c_req, - } - - return resp -} - func (r *response) Header() http.Header { return r.header } func (r *response) Write(p []byte) (n int, err error) { - if !r.headerSent { + if !r.header_sent { r.WriteHeader(http.StatusOK) } @@ -64,12 +53,11 @@ func (r *response) Write(p []byte) (n int, err error) { } func (r *response) WriteHeader(code int) { - if r.headerSent { - // Note: explicitly using Stderr, as Stdout is our HTTP output. + if r.header_sent { nxt_go_warn("multiple response.WriteHeader calls") return } - r.headerSent = true + r.header_sent = true // Set a default Content-Type if _, hasType := r.header["Content-Type"]; !hasType { @@ -86,21 +74,21 @@ func (r *response) WriteHeader(code int) { } } - C.nxt_cgo_response_create(r.c_req, C.int(code), C.int(fields), + C.nxt_unit_response_init(r.c_req, C.uint16_t(code), C.uint32_t(fields), C.uint32_t(fields_size)) for k, vv := range r.header { for _, v := range vv { - C.nxt_cgo_response_add_field(r.c_req, str_ref(k), C.uint8_t(len(k)), + C.nxt_unit_response_add_field(r.c_req, str_ref(k), C.uint8_t(len(k)), str_ref(v), C.uint32_t(len(v))) } } - C.nxt_cgo_response_send(r.c_req) + C.nxt_unit_response_send(r.c_req) } func (r *response) Flush() { - if !r.headerSent { + if !r.header_sent { r.WriteHeader(http.StatusOK) } } @@ -114,6 +102,6 @@ func wait_shm_ack(c chan int) { } //export nxt_go_shm_ack_handler -func nxt_go_shm_ack_handler() { +func nxt_go_shm_ack_handler(ctx *C.nxt_unit_ctx_t) { observer_registry_.notify(1) } diff --git a/go/unit.go b/go/unit.go index 1534479e..b5dd4f6c 100644 --- a/go/unit.go +++ b/go/unit.go @@ -30,15 +30,15 @@ func buf_ref(buf []byte) C.uintptr_t { return C.uintptr_t(uintptr(unsafe.Pointer(&buf[0]))) } -type StringHeader struct { +type string_header struct { Data unsafe.Pointer Len int } -func str_ref(s string) C.uintptr_t { - header := (*StringHeader)(unsafe.Pointer(&s)) +func str_ref(s string) *C.char { + header := (*string_header)(unsafe.Pointer(&s)) - return C.uintptr_t(uintptr(unsafe.Pointer(header.Data))) + return (*C.char)(header.Data) } func (buf *cbuf) init_bytes(b []byte) { @@ -46,12 +46,7 @@ func (buf *cbuf) init_bytes(b []byte) { buf.s = C.size_t(len(b)) } -func (buf *cbuf) init_string(s string) { - buf.b = str_ref(s) - buf.s = C.size_t(len(s)) -} - -type SliceHeader struct { +type slice_header struct { Data unsafe.Pointer Len int Cap int @@ -63,17 +58,17 @@ func (buf *cbuf) GoBytes() []byte { return b[:0] } - bytesHeader := &SliceHeader{ + header := &slice_header{ Data: unsafe.Pointer(uintptr(buf.b)), Len: int(buf.s), Cap: int(buf.s), } - return *(*[]byte)(unsafe.Pointer(bytesHeader)) + return *(*[]byte)(unsafe.Pointer(header)) } func GoBytes(buf unsafe.Pointer, size C.int) []byte { - bytesHeader := &SliceHeader{ + bytesHeader := &slice_header{ Data: buf, Len: int(size), Cap: int(size), @@ -82,12 +77,25 @@ func GoBytes(buf unsafe.Pointer, size C.int) []byte { return *(*[]byte)(unsafe.Pointer(bytesHeader)) } +func GoStringN(sptr *C.nxt_unit_sptr_t, l C.int) string { + p := unsafe.Pointer(sptr) + b := uintptr(p) + uintptr(*(*C.uint32_t)(p)) + + return C.GoStringN((*C.char)(unsafe.Pointer(b)), l) +} + func nxt_go_warn(format string, args ...interface{}) { str := fmt.Sprintf("[go] " + format, args...) C.nxt_cgo_warn(str_ref(str), C.uint32_t(len(str))) } +func nxt_go_alert(format string, args ...interface{}) { + str := fmt.Sprintf("[go] " + format, args...) + + C.nxt_cgo_alert(str_ref(str), C.uint32_t(len(str))) +} + type handler_registry struct { sync.RWMutex next uintptr diff --git a/src/nxt_unit.c b/src/nxt_unit.c index f0c68374..44525d04 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -54,12 +54,13 @@ static int nxt_unit_read_env(nxt_unit_port_t *ready_port, int *log_fd, uint32_t *stream, uint32_t *shm_limit); static int nxt_unit_ready(nxt_unit_ctx_t *ctx, int ready_fd, uint32_t stream, int queue_fd); -static int nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf); +static int nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf, + nxt_unit_request_info_t **preq); static int nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg); static int nxt_unit_ctx_ready(nxt_unit_ctx_t *ctx); static int nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, - nxt_unit_recv_msg_t *recv_msg); + nxt_unit_recv_msg_t *recv_msg, nxt_unit_request_info_t **preq); static int nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg); static int nxt_unit_request_check_response_port(nxt_unit_request_info_t *req, @@ -904,7 +905,8 @@ nxt_unit_ready(nxt_unit_ctx_t *ctx, int ready_fd, uint32_t stream, int queue_fd) static int -nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) +nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf, + nxt_unit_request_info_t **preq) { int rc; pid_t pid; @@ -1040,7 +1042,7 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) break; case _NXT_PORT_MSG_REQ_HEADERS: - rc = nxt_unit_process_req_headers(ctx, &recv_msg); + rc = nxt_unit_process_req_headers(ctx, &recv_msg, preq); break; case _NXT_PORT_MSG_REQ_BODY: @@ -1213,7 +1215,8 @@ nxt_unit_ctx_ready(nxt_unit_ctx_t *ctx) static int -nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) +nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, + nxt_unit_request_info_t **preq) { int res; nxt_unit_impl_t *lib; @@ -1329,7 +1332,12 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) } } - lib->callbacks.request_handler(req); + if (preq == NULL) { + lib->callbacks.request_handler(req); + + } else { + *preq = req; + } } return NXT_UNIT_OK; @@ -2179,7 +2187,8 @@ nxt_unit_response_add_field(nxt_unit_request_info_t *req, resp = req->response; if (nxt_slow_path(resp->fields_count >= req->response_max_fields)) { - nxt_unit_req_warn(req, "add_field: too many response fields"); + nxt_unit_req_warn(req, "add_field: too many response fields (%d)", + (int) resp->fields_count); return NXT_UNIT_ERROR; } @@ -2356,6 +2365,8 @@ nxt_unit_response_buf_alloc(nxt_unit_request_info_t *req, uint32_t size) if (nxt_slow_path(rc != NXT_UNIT_OK)) { nxt_unit_mmap_buf_release(mmap_buf); + nxt_unit_req_alert(req, "response_buf_alloc: failed to get out buf"); + return NULL; } @@ -4537,7 +4548,7 @@ nxt_unit_run_once_impl(nxt_unit_ctx_t *ctx) return rc; } - rc = nxt_unit_process_msg(ctx, rbuf); + rc = nxt_unit_process_msg(ctx, rbuf, NULL); if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { return NXT_UNIT_ERROR; } @@ -4686,7 +4697,7 @@ nxt_unit_process_pending_rbuf(nxt_unit_ctx_t *ctx) nxt_queue_each(rbuf, &pending_rbuf, nxt_unit_read_buf_t, link) { if (nxt_fast_path(rc != NXT_UNIT_ERROR)) { - rc = nxt_unit_process_msg(&ctx_impl->ctx, rbuf); + rc = nxt_unit_process_msg(&ctx_impl->ctx, rbuf, NULL); } else { nxt_unit_read_buf_release(ctx, rbuf); @@ -4793,7 +4804,7 @@ nxt_unit_run_ctx(nxt_unit_ctx_t *ctx) goto retry; } - rc = nxt_unit_process_msg(ctx, rbuf); + rc = nxt_unit_process_msg(ctx, rbuf, NULL); if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { break; } @@ -4902,7 +4913,7 @@ nxt_unit_run_shared(nxt_unit_ctx_t *ctx) break; } - rc = nxt_unit_process_msg(ctx, rbuf); + rc = nxt_unit_process_msg(ctx, rbuf, NULL); if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { break; } @@ -4921,6 +4932,46 @@ nxt_unit_run_shared(nxt_unit_ctx_t *ctx) } +nxt_unit_request_info_t * +nxt_unit_dequeue_request(nxt_unit_ctx_t *ctx) +{ + int rc; + nxt_unit_impl_t *lib; + nxt_unit_read_buf_t *rbuf; + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_request_info_t *req; + + nxt_unit_ctx_use(ctx); + + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + req = NULL; + + if (nxt_slow_path(!ctx_impl->online)) { + goto done; + } + + rbuf = nxt_unit_read_buf_get(ctx); + if (nxt_slow_path(rbuf == NULL)) { + goto done; + } + + rc = nxt_unit_app_queue_recv(lib->shared_port, rbuf); + if (rc != NXT_UNIT_OK) { + goto done; + } + + (void) nxt_unit_process_msg(ctx, rbuf, &req); + +done: + + nxt_unit_ctx_release(ctx); + + return req; +} + + int nxt_unit_is_main_ctx(nxt_unit_ctx_t *ctx) { @@ -4977,7 +5028,7 @@ retry: return rc; } - rc = nxt_unit_process_msg(ctx, rbuf); + rc = nxt_unit_process_msg(ctx, rbuf, NULL); if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { return NXT_UNIT_ERROR; } diff --git a/src/nxt_unit.h b/src/nxt_unit.h index 90cba2a3..1e1a8dbe 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -213,6 +213,8 @@ int nxt_unit_run_ctx(nxt_unit_ctx_t *ctx); int nxt_unit_run_shared(nxt_unit_ctx_t *ctx); +nxt_unit_request_info_t *nxt_unit_dequeue_request(nxt_unit_ctx_t *ctx); + int nxt_unit_is_main_ctx(nxt_unit_ctx_t *ctx); /* -- cgit From 300347a5cffa4187921384bdd02c5bb90875f9e5 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 18 Nov 2020 22:33:53 +0300 Subject: Libunit: making minor tweaks. Removing unnecessary context operations from shared queue processing loop. Initializing temporary queues only when required. --- src/nxt_unit.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 44525d04..e74c8370 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -4675,8 +4675,6 @@ nxt_unit_process_pending_rbuf(nxt_unit_ctx_t *ctx) nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_read_buf_t *rbuf; - nxt_queue_init(&pending_rbuf); - ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); pthread_mutex_lock(&ctx_impl->mutex); @@ -4687,6 +4685,8 @@ nxt_unit_process_pending_rbuf(nxt_unit_ctx_t *ctx) return NXT_UNIT_OK; } + nxt_queue_init(&pending_rbuf); + nxt_queue_add(&pending_rbuf, &ctx_impl->pending_rbuf); nxt_queue_init(&ctx_impl->pending_rbuf); @@ -4719,8 +4719,6 @@ nxt_unit_process_ready_req(nxt_unit_ctx_t *ctx) nxt_unit_request_info_t *req; nxt_unit_request_info_impl_t *req_impl; - nxt_queue_init(&ready_req); - ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); pthread_mutex_lock(&ctx_impl->mutex); @@ -4731,6 +4729,8 @@ nxt_unit_process_ready_req(nxt_unit_ctx_t *ctx) return; } + nxt_queue_init(&ready_req); + nxt_queue_add(&ready_req, &ctx_impl->ready_req); nxt_queue_init(&ctx_impl->ready_req); @@ -4917,13 +4917,6 @@ nxt_unit_run_shared(nxt_unit_ctx_t *ctx) if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { break; } - - rc = nxt_unit_process_pending_rbuf(ctx); - if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { - break; - } - - nxt_unit_process_ready_req(ctx); } nxt_unit_ctx_release(ctx); -- cgit From 6c3c83561a97b91f18a771e0c582c5ed4013a9a6 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 18 Nov 2020 22:33:53 +0300 Subject: Libunit: closing active requests on quit. --- src/nodejs/unit-http/unit.cpp | 2 ++ src/nxt_unit.c | 37 +++++++++++++++++++++------- src/python/nxt_python_asgi.c | 15 ++++++++++- src/python/nxt_python_asgi.h | 1 + src/python/nxt_python_asgi_http.c | 52 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 96 insertions(+), 11 deletions(-) diff --git a/src/nodejs/unit-http/unit.cpp b/src/nodejs/unit-http/unit.cpp index 1ee5b742..c5bca49a 100644 --- a/src/nodejs/unit-http/unit.cpp +++ b/src/nodejs/unit-http/unit.cpp @@ -307,6 +307,8 @@ Unit::close_handler(nxt_unit_request_info_t *req) } catch (exception &e) { nxt_unit_req_warn(req, "close_handler: %s", e.str); + nxt_unit_request_done(req, NXT_UNIT_ERROR); + return; } diff --git a/src/nxt_unit.c b/src/nxt_unit.c index e74c8370..69948954 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -1643,8 +1643,6 @@ nxt_unit_process_websocket(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) } if (recv_msg->last) { - req_impl->websocket = 0; - if (cb->close_handler) { nxt_unit_req_debug(req, "close_handler"); @@ -1737,8 +1735,6 @@ nxt_unit_request_info_release(nxt_unit_request_info_t *req) nxt_unit_request_hash_find(req->ctx, req_impl->stream, 1); } - req_impl->websocket = 0; - while (req_impl->outgoing_buf != NULL) { nxt_unit_mmap_buf_free(req_impl->outgoing_buf); } @@ -5708,9 +5704,12 @@ nxt_unit_remove_process(nxt_unit_impl_t *lib, nxt_unit_process_t *process) static void nxt_unit_quit(nxt_unit_ctx_t *ctx) { - nxt_port_msg_t msg; - nxt_unit_impl_t *lib; - nxt_unit_ctx_impl_t *ctx_impl; + nxt_port_msg_t msg; + nxt_unit_impl_t *lib; + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_callbacks_t *cb; + nxt_unit_request_info_t *req; + nxt_unit_request_info_impl_t *req_impl; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); @@ -5721,10 +5720,30 @@ nxt_unit_quit(nxt_unit_ctx_t *ctx) ctx_impl->online = 0; - if (lib->callbacks.quit != NULL) { - lib->callbacks.quit(ctx); + cb = &lib->callbacks; + + if (cb->quit != NULL) { + cb->quit(ctx); } + nxt_queue_each(req_impl, &ctx_impl->active_req, + nxt_unit_request_info_impl_t, link) + { + req = &req_impl->req; + + nxt_unit_req_warn(req, "active request on ctx quit"); + + if (cb->close_handler) { + nxt_unit_req_debug(req, "close_handler"); + + cb->close_handler(req); + + } else { + nxt_unit_request_done(req, NXT_UNIT_ERROR); + } + + } nxt_queue_loop; + if (ctx != &lib->main_ctx.ctx) { return; } diff --git a/src/python/nxt_python_asgi.c b/src/python/nxt_python_asgi.c index e11a3b6c..98aeedf4 100644 --- a/src/python/nxt_python_asgi.c +++ b/src/python/nxt_python_asgi.c @@ -25,6 +25,7 @@ static int nxt_python_asgi_run(nxt_unit_ctx_t *ctx); static void nxt_py_asgi_remove_reader(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port); static void nxt_py_asgi_request_handler(nxt_unit_request_info_t *req); +static void nxt_py_asgi_close_handler(nxt_unit_request_info_t *req); static PyObject *nxt_py_asgi_create_http_scope(nxt_unit_request_info_t *req); static PyObject *nxt_py_asgi_create_address(nxt_unit_sptr_t *sptr, uint8_t len, @@ -179,7 +180,7 @@ nxt_python_asgi_init(nxt_unit_init_t *init, nxt_python_proto_t *proto) init->callbacks.request_handler = nxt_py_asgi_request_handler; init->callbacks.data_handler = nxt_py_asgi_http_data_handler; init->callbacks.websocket_handler = nxt_py_asgi_websocket_handler; - init->callbacks.close_handler = nxt_py_asgi_websocket_close_handler; + init->callbacks.close_handler = nxt_py_asgi_close_handler; init->callbacks.quit = nxt_py_asgi_quit; init->callbacks.shm_ack_handler = nxt_py_asgi_shm_ack_handler; init->callbacks.add_port = nxt_py_asgi_add_port; @@ -551,6 +552,18 @@ release_asgi: } +static void +nxt_py_asgi_close_handler(nxt_unit_request_info_t *req) +{ + if (req->request->websocket_handshake) { + nxt_py_asgi_websocket_close_handler(req); + + } else { + nxt_py_asgi_http_close_handler(req); + } +} + + static PyObject * nxt_py_asgi_create_http_scope(nxt_unit_request_info_t *req) { diff --git a/src/python/nxt_python_asgi.h b/src/python/nxt_python_asgi.h index c3c3e17a..37f2a099 100644 --- a/src/python/nxt_python_asgi.h +++ b/src/python/nxt_python_asgi.h @@ -58,6 +58,7 @@ int nxt_py_asgi_http_init(void); PyObject *nxt_py_asgi_http_create(nxt_unit_request_info_t *req); void nxt_py_asgi_http_data_handler(nxt_unit_request_info_t *req); int nxt_py_asgi_http_drain(nxt_queue_link_t *lnk); +void nxt_py_asgi_http_close_handler(nxt_unit_request_info_t *req); int nxt_py_asgi_websocket_init(void); PyObject *nxt_py_asgi_websocket_create(nxt_unit_request_info_t *req); diff --git a/src/python/nxt_python_asgi_http.c b/src/python/nxt_python_asgi_http.c index a5034ea6..5ea8e0a5 100644 --- a/src/python/nxt_python_asgi_http.c +++ b/src/python/nxt_python_asgi_http.c @@ -24,6 +24,7 @@ typedef struct { uint64_t content_length; uint64_t bytes_sent; int complete; + int closed; PyObject *send_body; Py_ssize_t send_body_off; } nxt_py_asgi_http_t; @@ -94,6 +95,7 @@ nxt_py_asgi_http_create(nxt_unit_request_info_t *req) http->content_length = -1; http->bytes_sent = 0; http->complete = 0; + http->closed = 0; http->send_body = NULL; http->send_body_off = 0; } @@ -115,7 +117,13 @@ nxt_py_asgi_http_receive(PyObject *self, PyObject *none) nxt_unit_req_debug(req, "asgi_http_receive"); - msg = nxt_py_asgi_http_read_msg(http); + if (nxt_slow_path(http->closed || nxt_unit_response_is_sent(req))) { + msg = nxt_py_asgi_new_msg(req, nxt_py_http_disconnect_str); + + } else { + msg = nxt_py_asgi_http_read_msg(http); + } + if (nxt_slow_path(msg == NULL)) { return NULL; } @@ -561,6 +569,48 @@ fail: } +void +nxt_py_asgi_http_close_handler(nxt_unit_request_info_t *req) +{ + PyObject *msg, *future, *res; + nxt_py_asgi_http_t *http; + + http = req->data; + + nxt_unit_req_debug(req, "asgi_http_close_handler"); + + http->closed = 1; + + if (http->receive_future == NULL) { + return; + } + + msg = nxt_py_asgi_new_msg(req, nxt_py_http_disconnect_str); + if (nxt_slow_path(msg == NULL)) { + return; + } + + if (msg == Py_None) { + Py_DECREF(msg); + return; + } + + future = http->receive_future; + http->receive_future = NULL; + + res = PyObject_CallMethodObjArgs(future, nxt_py_set_result_str, msg, NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_req_alert(req, "'set_result' call failed"); + nxt_python_print_exception(); + } + + Py_XDECREF(res); + Py_DECREF(future); + + Py_DECREF(msg); +} + + static PyObject * nxt_py_asgi_http_done(PyObject *self, PyObject *future) { -- cgit From 66bb41e8bbe81d82a66f0d7188ad89963b4cd251 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 18 Nov 2020 22:33:53 +0300 Subject: Libunit: fixing read buffer allocations on exit. --- src/nxt_unit.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 69948954..0790afc4 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -5029,12 +5029,12 @@ retry: nxt_unit_process_ready_req(ctx); - rbuf = nxt_unit_read_buf_get(ctx); - if (nxt_slow_path(rbuf == NULL)) { - return NXT_UNIT_ERROR; - } - if (ctx_impl->online) { + rbuf = nxt_unit_read_buf_get(ctx); + if (nxt_slow_path(rbuf == NULL)) { + return NXT_UNIT_ERROR; + } + goto retry; } -- cgit From 25219a7ece30a5b21ac0674c557137f91b4b47fe Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 18 Nov 2020 22:33:53 +0300 Subject: Python: improving ASGI http send message processing. --- src/python/nxt_python_asgi_http.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/python/nxt_python_asgi_http.c b/src/python/nxt_python_asgi_http.c index 5ea8e0a5..d88c4b00 100644 --- a/src/python/nxt_python_asgi_http.c +++ b/src/python/nxt_python_asgi_http.c @@ -262,22 +262,23 @@ nxt_py_asgi_http_send(PyObject *self, PyObject *dict) nxt_unit_req_debug(http->req, "asgi_http_send type is '%.*s'", (int) type_len, type_str); - if (type_len == (Py_ssize_t) response_start.length - && memcmp(type_str, response_start.start, type_len) == 0) - { - return nxt_py_asgi_http_response_start(http, dict); - } + if (nxt_unit_response_is_init(http->req)) { + if (nxt_str_eq(&response_body, type_str, (size_t) type_len)) { + return nxt_py_asgi_http_response_body(http, dict); + } - if (type_len == (Py_ssize_t) response_body.length - && memcmp(type_str, response_body.start, type_len) == 0) - { - return nxt_py_asgi_http_response_body(http, dict); + return PyErr_Format(PyExc_RuntimeError, + "Expected ASGI message 'http.response.body', " + "but got '%U'", type); } - nxt_unit_req_error(http->req, "asgi_http_send: unexpected 'type': '%.*s'", - (int) type_len, type_str); + if (nxt_str_eq(&response_start, type_str, (size_t) type_len)) { + return nxt_py_asgi_http_response_start(http, dict); + } - return PyErr_Format(PyExc_AssertionError, "unexpected 'type': '%U'", type); + return PyErr_Format(PyExc_RuntimeError, + "Expected ASGI message 'http.response.start', " + "but got '%U'", type); } -- cgit From e154d7a3a26359e60544b2b578990fc3514422a4 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Thu, 19 Nov 2020 05:21:22 +0000 Subject: Tests: style. --- test/test_http_header.py | 76 +++++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/test/test_http_header.py b/test/test_http_header.py index 8381a0d9..b1c77066 100644 --- a/test/test_http_header.py +++ b/test/test_http_header.py @@ -154,54 +154,58 @@ Connection: close def test_http_header_field_leading_sp(self): self.load('empty') - resp = self.get( - headers={ - 'Host': 'localhost', - ' Custom-Header': 'blah', - 'Connection': 'close', - } - ) - - assert resp['status'] == 400, 'field leading sp' + assert ( + self.get( + headers={ + 'Host': 'localhost', + ' Custom-Header': 'blah', + 'Connection': 'close', + } + )['status'] + == 400 + ), 'field leading sp' def test_http_header_field_leading_htab(self): self.load('empty') - resp = self.get( - headers={ - 'Host': 'localhost', - '\tCustom-Header': 'blah', - 'Connection': 'close', - } - ) - - assert resp['status'] == 400, 'field leading htab' + assert ( + self.get( + headers={ + 'Host': 'localhost', + '\tCustom-Header': 'blah', + 'Connection': 'close', + } + )['status'] + == 400 + ), 'field leading htab' def test_http_header_field_trailing_sp(self): self.load('empty') - resp = self.get( - headers={ - 'Host': 'localhost', - 'Custom-Header ': 'blah', - 'Connection': 'close', - } - ) - - assert resp['status'] == 400, 'field trailing sp' + assert ( + self.get( + headers={ + 'Host': 'localhost', + 'Custom-Header ': 'blah', + 'Connection': 'close', + } + )['status'] + == 400 + ), 'field trailing sp' def test_http_header_field_trailing_htab(self): self.load('empty') - resp = self.get( - headers={ - 'Host': 'localhost', - 'Custom-Header\t': 'blah', - 'Connection': 'close', - } - ) - - assert resp['status'] == 400, 'field trailing htab' + assert ( + self.get( + headers={ + 'Host': 'localhost', + 'Custom-Header\t': 'blah', + 'Connection': 'close', + } + )['status'] + == 400 + ), 'field trailing htab' def test_http_header_content_length_big(self): self.load('empty') -- cgit From 18ddb747725d24a65b61e0b8ca5d072f52724190 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Thu, 19 Nov 2020 05:21:48 +0000 Subject: Tests: added tests for a "discard_unsafe_fields" option. --- test/python/header_fields/wsgi.py | 9 +++++++++ test/test_http_header.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 test/python/header_fields/wsgi.py diff --git a/test/python/header_fields/wsgi.py b/test/python/header_fields/wsgi.py new file mode 100644 index 00000000..bd1ba0e2 --- /dev/null +++ b/test/python/header_fields/wsgi.py @@ -0,0 +1,9 @@ +def application(environ, start_response): + + h = (k for k, v in environ.items() if k.startswith('HTTP_')) + + start_response('200', [ + ('Content-Length', '0'), + ('All-Headers', ','.join(h)) + ]) + return [] diff --git a/test/test_http_header.py b/test/test_http_header.py index b1c77066..fdb557cf 100644 --- a/test/test_http_header.py +++ b/test/test_http_header.py @@ -431,3 +431,41 @@ Connection: close )['status'] == 400 ), 'Host multiple fields' + + def test_http_discard_unsafe_fields(self): + self.load('header_fields') + + def check_status(header): + resp = self.get( + headers={ + 'Host': 'localhost', + header: 'blah', + 'Connection': 'close', + } + ) + + assert resp['status'] == 200 + return resp + + resp = check_status("!Custom-Header") + assert 'CUSTOM' not in resp['headers']['All-Headers'] + + resp = check_status("Custom_Header") + assert 'CUSTOM' not in resp['headers']['All-Headers'] + + assert 'success' in self.conf( + {'http': {'discard_unsafe_fields': False}}, 'settings', + ) + + resp = check_status("!#$%&'*+.^`|~Custom_Header") + assert 'CUSTOM' in resp['headers']['All-Headers'] + + assert 'success' in self.conf( + {'http': {'discard_unsafe_fields': True}}, 'settings', + ) + + resp = check_status("!Custom-Header") + assert 'CUSTOM' not in resp['headers']['All-Headers'] + + resp = check_status("Custom_Header") + assert 'CUSTOM' not in resp['headers']['All-Headers'] -- cgit From 2a381a82a6e1bc2bd5d2f43a08fce50a1994f2e8 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 19 Nov 2020 13:49:12 +0300 Subject: Libunit: fixing read buffer leakage. If shared queue is empty, allocated read buffer should be explicitly released. Found by Coverity (CID 363943). The issue was introduced in f5ba5973a0a3. --- src/nxt_unit.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 0790afc4..097f50d6 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -4948,6 +4948,7 @@ nxt_unit_dequeue_request(nxt_unit_ctx_t *ctx) rc = nxt_unit_app_queue_recv(lib->shared_port, rbuf); if (rc != NXT_UNIT_OK) { + nxt_unit_read_buf_release(ctx, rbuf); goto done; } -- cgit From e3af18834d7cc32734cba7532d8864bb343b416b Mon Sep 17 00:00:00 2001 From: Axel Duch Date: Tue, 17 Nov 2020 15:03:30 +0000 Subject: Router: matching regular expressions support. --- auto/help | 2 + auto/options | 10 ++- auto/pcre | 56 ++++++++++++---- auto/sources | 11 ++++ configure | 6 +- src/nxt_conf_validation.c | 35 +++++++++- src/nxt_http.h | 6 ++ src/nxt_http_route.c | 99 ++++++++++++++++++++++++---- src/nxt_pcre.c | 135 ++++++++++++++++++++++++++++++++++++++ src/nxt_pcre2.c | 161 ++++++++++++++++++++++++++++++++++++++++++++++ src/nxt_regex.h | 41 ++++++++++++ src/nxt_runtime.c | 4 +- test/test_routing.py | 42 ++++++++++++ 13 files changed, 572 insertions(+), 36 deletions(-) create mode 100644 src/nxt_pcre.c create mode 100644 src/nxt_pcre2.c create mode 100644 src/nxt_regex.h diff --git a/auto/help b/auto/help index f5f10010..31c68567 100644 --- a/auto/help +++ b/auto/help @@ -35,6 +35,8 @@ cat << END --no-ipv6 disable IPv6 support --no-unix-sockets disable Unix domain sockets support + --no-regex disable regular expression support + --no-pcre2 force using PCRE library --openssl enable OpenSSL library usage diff --git a/auto/options b/auto/options index d315b227..b6007bc2 100644 --- a/auto/options +++ b/auto/options @@ -16,8 +16,11 @@ NXT_DEBUG=NO NXT_INET6=YES NXT_UNIX_DOMAIN=YES -NXT_REGEX=NO -NXT_PCRE=NO +NXT_PCRE_CFLAGS= +NXT_PCRE_LIB= + +NXT_REGEX=YES +NXT_TRY_PCRE2=YES NXT_TLS=NO NXT_OPENSSL=NO @@ -73,7 +76,8 @@ do --no-ipv6) NXT_INET6=NO ;; --no-unix-sockets) NXT_UNIX_DOMAIN=NO ;; - --pcre) NXT_PCRE=YES ;; + --no-regex) NXT_REGEX=NO ;; + --no-pcre2) NXT_TRY_PCRE2=NO ;; --openssl) NXT_OPENSSL=YES ;; --gnutls) NXT_GNUTLS=YES ;; diff --git a/auto/pcre b/auto/pcre index e8765cef..955e4baf 100644 --- a/auto/pcre +++ b/auto/pcre @@ -3,15 +3,41 @@ # Copyright (C) NGINX, Inc. -NXT_REGEX=NO -NXT_PCRE_CFLAGS= -NXT_PCRE_LIB= +nxt_found=no +NXT_HAVE_PCRE2=NO +if [ $NXT_TRY_PCRE2 = YES ]; then + if /bin/sh -c "(pcre2-config --version)" >> $NXT_AUTOCONF_ERR 2>&1; then -if [ $NXT_PCRE = YES ]; then + NXT_PCRE_CFLAGS=`pcre2-config --cflags` + NXT_PCRE_LIB=`pcre2-config --libs8` - nxt_found=no + nxt_feature="PCRE2 library" + nxt_feature_name=NXT_HAVE_PCRE2 + nxt_feature_run=no + nxt_feature_incs="-DPCRE2_CODE_UNIT_WIDTH=8 $NXT_PCRE_CFLAGS" + nxt_feature_libs=$NXT_PCRE_LIB + nxt_feature_test="#include + + int main(void) { + pcre2_code *re; + + re = pcre2_compile((PCRE2_SPTR)\"\", + PCRE2_ZERO_TERMINATED, 0, + NULL, NULL, NULL); + return (re == NULL); + }" + + . auto/feature + + if [ $nxt_found = yes ]; then + NXT_HAVE_PCRE2=YES + $echo " + PCRE2 version: `pcre2-config --version`" + fi + fi +fi +if [ $nxt_found = no ]; then if /bin/sh -c "(pcre-config --version)" >> $NXT_AUTOCONF_ERR 2>&1; then NXT_PCRE_CFLAGS=`pcre-config --cflags` @@ -33,17 +59,19 @@ if [ $NXT_PCRE = YES ]; then return 0; }" . auto/feature - fi - if [ $nxt_found = no ]; then - $echo - $echo $0: error: no PCRE library found. - $echo - exit 1; + if [ $nxt_found = yes ]; then + $echo " + PCRE version: `pcre-config --version`" + fi fi +fi - NXT_REGEX=YES - nxt_have=NXT_REGEX . auto/have +if [ $nxt_found = yes ]; then + nxt_have=NXT_HAVE_REGEX . auto/have - $echo " + PCRE version: `pcre-config --version`" +else + $echo + $echo $0: error: no PCRE library found. + $echo + exit 1; fi diff --git a/auto/sources b/auto/sources index e44dc4bb..01fec6c1 100644 --- a/auto/sources +++ b/auto/sources @@ -128,6 +128,9 @@ NXT_LIB_GNUTLS_SRCS="src/nxt_gnutls.c" NXT_LIB_CYASSL_SRCS="src/nxt_cyassl.c" NXT_LIB_POLARSSL_SRCS="src/nxt_polarssl.c" +NXT_LIB_PCRE_SRCS="src/nxt_pcre.c" +NXT_LIB_PCRE2_SRCS="src/nxt_pcre2.c" + NXT_LIB_EPOLL_SRCS="src/nxt_epoll_engine.c" NXT_LIB_KQUEUE_SRCS="src/nxt_kqueue_engine.c" NXT_LIB_EVENTPORT_SRCS="src/nxt_eventport_engine.c" @@ -211,6 +214,14 @@ if [ $NXT_POLARSSL = YES ]; then fi +if [ "$NXT_REGEX" = "YES" ]; then + if [ "$NXT_HAVE_PCRE2" = "YES" ]; then + NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_PCRE2_SRCS" + else + NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_PCRE_SRCS" + fi +fi + if [ "$NXT_HAVE_EPOLL" = "YES" -o "$NXT_TEST_BUILD_EPOLL" = "YES" ]; then NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_EPOLL_SRCS" fi diff --git a/configure b/configure index c67e4728..ece4f12a 100755 --- a/configure +++ b/configure @@ -127,7 +127,11 @@ NXT_LIBRT= . auto/unix . auto/os/conf . auto/ssltls -. auto/pcre + +if [ $NXT_REGEX = YES ]; then + . auto/pcre +fi + . auto/isolation . auto/capability diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index f4e7c358..acb2e3de 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -11,6 +11,7 @@ #include #include #include +#include typedef enum { @@ -1504,8 +1505,12 @@ static nxt_int_t nxt_conf_vldt_match_pattern(nxt_conf_validation_t *vldt, nxt_conf_value_t *value) { - nxt_str_t pattern; - nxt_uint_t i, first, last; + nxt_str_t pattern; + nxt_uint_t i, first, last; +#if (NXT_HAVE_REGEX) + nxt_regex_t *re; + nxt_regex_err_t err; +#endif if (nxt_conf_type(value) != NXT_CONF_STRING) { return nxt_conf_vldt_error(vldt, "The \"match\" patterns for \"host\", " @@ -1519,6 +1524,32 @@ nxt_conf_vldt_match_pattern(nxt_conf_validation_t *vldt, } first = (pattern.start[0] == '!'); + + if (first < pattern.length && pattern.start[first] == '~') { +#if (NXT_HAVE_REGEX) + pattern.start += first + 1; + pattern.length -= first + 1; + + re = nxt_regex_compile(vldt->pool, &pattern, &err); + if (nxt_slow_path(re == NULL)) { + if (err.offset < pattern.length) { + return nxt_conf_vldt_error(vldt, "Invalid regular expression: " + "%s at offset %d", + err.msg, err.offset); + } + + return nxt_conf_vldt_error(vldt, "Invalid regular expression: %s", + err.msg); + } + + return NXT_OK; +#else + return nxt_conf_vldt_error(vldt, "Unit is built without support of " + "regular expressions: \"--no-regex\" " + "./configure option was set."); +#endif + } + last = pattern.length - 1; for (i = first; i < last; i++) { diff --git a/src/nxt_http.h b/src/nxt_http.h index 08181520..1418be95 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -7,6 +7,8 @@ #ifndef _NXT_HTTP_H_INCLUDED_ #define _NXT_HTTP_H_INCLUDED_ +#include + typedef enum { NXT_HTTP_UNSET = -1, @@ -168,6 +170,10 @@ struct nxt_http_request_s { void *req_rpc_data; +#if (NXT_HAVE_REGEX) + nxt_regex_match_t *regex_match; +#endif + nxt_http_peer_t *peer; nxt_buf_t *last; diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index ae91076a..9aaa708e 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -8,6 +8,7 @@ #include #include #include +#include typedef enum { @@ -76,12 +77,20 @@ typedef struct { typedef struct { + union { + nxt_array_t *pattern_slices; +#if (NXT_HAVE_REGEX) + nxt_regex_t *regex; +#endif + } u; uint32_t min_length; - nxt_array_t *pattern_slices; uint8_t case_sensitive; /* 1 bit */ uint8_t negative; /* 1 bit */ uint8_t any; /* 1 bit */ +#if (NXT_HAVE_REGEX) + uint8_t regex; /* 1 bit */ +#endif } nxt_http_route_pattern_t; @@ -1060,24 +1069,24 @@ nxt_http_route_pattern_create(nxt_task_t *task, nxt_mp_t *mp, nxt_str_t test, tmp; nxt_int_t ret; nxt_array_t *slices; +#if (NXT_HAVE_REGEX) + nxt_regex_t *re; + nxt_regex_err_t err; +#endif nxt_http_route_pattern_type_t type; - nxt_http_route_pattern_slice_t *slice; type = NXT_HTTP_ROUTE_PATTERN_EXACT; nxt_conf_get_string(cv, &test); - slices = nxt_array_create(mp, 1, sizeof(nxt_http_route_pattern_slice_t)); - if (nxt_slow_path(slices == NULL)) { - return NXT_ERROR; - } - - pattern->pattern_slices = slices; - + pattern->u.pattern_slices = NULL; pattern->negative = 0; pattern->any = 1; pattern->min_length = 0; +#if (NXT_HAVE_REGEX) + pattern->regex = 0; +#endif if (test.length != 0 && test.start[0] == '!') { test.start++; @@ -1087,6 +1096,41 @@ nxt_http_route_pattern_create(nxt_task_t *task, nxt_mp_t *mp, pattern->any = 0; } + if (test.length > 0 && test.start[0] == '~') { +#if (NXT_HAVE_REGEX) + test.start++; + test.length--; + + re = nxt_regex_compile(mp, &test, &err); + if (nxt_slow_path(re == NULL)) { + if (err.offset < test.length) { + nxt_alert(task, "nxt_regex_compile(%V) failed: %s at offset %d", + &test, err.msg, (int) err.offset); + return NXT_ERROR; + } + + nxt_alert(task, "nxt_regex_compile(%V) failed %s", &test, err.msg); + + return NXT_ERROR; + } + + pattern->u.regex = re; + pattern->regex = 1; + + return NXT_OK; + +#else + return NXT_ERROR; +#endif + } + + slices = nxt_array_create(mp, 1, sizeof(nxt_http_route_pattern_slice_t)); + if (nxt_slow_path(slices == NULL)) { + return NXT_ERROR; + } + + pattern->u.pattern_slices = slices; + if (test.length == 0) { slice = nxt_array_add(slices); if (nxt_slow_path(slice == NULL)) { @@ -1980,6 +2024,9 @@ nxt_http_route_header(nxt_http_request_t *r, nxt_http_route_rule_t *rule) } ret = nxt_http_route_test_rule(r, rule, f->value, f->value_length); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } if (ret == 0) { return ret; @@ -2155,7 +2202,7 @@ static nxt_int_t nxt_http_route_test_argument(nxt_http_request_t *r, nxt_http_route_rule_t *rule, nxt_array_t *array) { - nxt_bool_t ret; + nxt_int_t ret; nxt_http_name_value_t *nv, *end; ret = 0; @@ -2171,6 +2218,10 @@ nxt_http_route_test_argument(nxt_http_request_t *r, { ret = nxt_http_route_test_rule(r, rule, nv->value, nv->value_length); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } + if (ret == 0) { break; } @@ -2189,7 +2240,7 @@ nxt_http_route_scheme(nxt_http_request_t *r, nxt_http_route_rule_t *rule) nxt_bool_t tls, https; nxt_http_route_pattern_slice_t *pattern_slice; - pattern_slice = rule->pattern[0].pattern_slices->elts; + pattern_slice = rule->pattern[0].u.pattern_slices->elts; https = (pattern_slice->length == nxt_length("https")); tls = (r->tls != NULL); @@ -2337,7 +2388,7 @@ static nxt_int_t nxt_http_route_test_cookie(nxt_http_request_t *r, nxt_http_route_rule_t *rule, nxt_array_t *array) { - nxt_bool_t ret; + nxt_int_t ret; nxt_http_name_value_t *nv, *end; ret = 0; @@ -2353,6 +2404,10 @@ nxt_http_route_test_cookie(nxt_http_request_t *r, { ret = nxt_http_route_test_rule(r, rule, nv->value, nv->value_length); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } + if (ret == 0) { break; } @@ -2378,6 +2433,9 @@ nxt_http_route_test_rule(nxt_http_request_t *r, nxt_http_route_rule_t *rule, while (pattern < end) { ret = nxt_http_route_pattern(r, pattern, start, length); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } /* nxt_http_route_pattern() returns either 1 or 0. */ ret ^= pattern->negative; @@ -2403,11 +2461,26 @@ nxt_http_route_pattern(nxt_http_request_t *r, nxt_http_route_pattern_t *pattern, nxt_array_t *pattern_slices; nxt_http_route_pattern_slice_t *pattern_slice; +#if (NXT_HAVE_REGEX) + if (pattern->regex) { + if (r->regex_match == NULL) { + r->regex_match = nxt_regex_match_create(r->mem_pool, 0); + if (nxt_slow_path(r->regex_match == NULL)) { + return NXT_ERROR; + } + } + + return nxt_regex_match(pattern->u.regex, start, length, r->regex_match); + } +#endif + if (length < pattern->min_length) { return 0; } - pattern_slices = pattern->pattern_slices; + nxt_assert(pattern->u.pattern_slices != NULL); + + pattern_slices = pattern->u.pattern_slices; pattern_slice = pattern_slices->elts; end = start + length; diff --git a/src/nxt_pcre.c b/src/nxt_pcre.c new file mode 100644 index 00000000..737fc7cf --- /dev/null +++ b/src/nxt_pcre.c @@ -0,0 +1,135 @@ +/* + * Copyright (C) Axel Duch + * Copyright (C) NGINX, Inc. + */ + +#include +#include +#include + + +struct nxt_regex_s { + pcre *code; + pcre_extra *extra; + nxt_str_t pattern; +}; + +struct nxt_regex_match_s { + int ovecsize; + int ovec[]; +}; + + +static void *nxt_pcre_malloc(size_t size); +static void nxt_pcre_free(void *p); + +static nxt_mp_t *nxt_pcre_mp; + + +nxt_regex_t * +nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, nxt_regex_err_t *err) +{ + int erroffset; + char *pattern; + void *saved_malloc, *saved_free; + nxt_regex_t *re; + + err->offset = source->length; + + re = nxt_mp_get(mp, sizeof(nxt_regex_t) + source->length + 1); + if (nxt_slow_path(re == NULL)) { + err->msg = "memory allocation failed"; + return NULL; + } + + pattern = nxt_pointer_to(re, sizeof(nxt_regex_t)); + + nxt_memcpy(pattern, source->start, source->length); + pattern[source->length] = '\0'; + + re->pattern.length = source->length; + re->pattern.start = (u_char *) pattern; + + saved_malloc = pcre_malloc; + saved_free = pcre_free; + + pcre_malloc = nxt_pcre_malloc; + pcre_free = nxt_pcre_free; + nxt_pcre_mp = mp; + + re->code = pcre_compile(pattern, 0, &err->msg, &erroffset, NULL); + if (nxt_fast_path(re->code != NULL)) { +#if 0 + re->extra = pcre_study(re->code, PCRE_STUDY_JIT_COMPILE, &err->msg); + if (nxt_slow_path(re->extra == NULL && err->msg != NULL)) { + nxt_log_warn(thr->log, "pcre_study(%V) failed: %s", source, err->msg); + } +#else + re->extra = NULL; +#endif + + } else { + err->offset = erroffset; + re = NULL; + } + + pcre_malloc = saved_malloc; + pcre_free = saved_free; + + return re; +} + + +static void* +nxt_pcre_malloc(size_t size) +{ + if (nxt_slow_path(nxt_pcre_mp == NULL)) { + nxt_thread_log_alert("pcre_malloc(%uz) called without memory pool", + size); + return NULL; + } + + nxt_thread_log_debug("pcre_malloc(%uz), pool %p", size, nxt_pcre_mp); + + return nxt_mp_get(nxt_pcre_mp, size); +} + + +static void +nxt_pcre_free(void *p) +{ +} + + +nxt_regex_match_t * +nxt_regex_match_create(nxt_mp_t *mp, size_t size) +{ + nxt_regex_match_t *match; + + match = nxt_mp_get(mp, sizeof(nxt_regex_match_t) + sizeof(int) * size); + if (nxt_fast_path(match != NULL)) { + match->ovecsize = size; + } + + return match; +} + + +nxt_int_t +nxt_regex_match(nxt_regex_t *re, u_char *subject, size_t length, + nxt_regex_match_t *match) +{ + int ret; + + ret = pcre_exec(re->code, re->extra, (const char *) subject, length, 0, 0, + match->ovec, match->ovecsize); + if (nxt_slow_path(ret < PCRE_ERROR_NOMATCH)) { + nxt_thread_log_error(NXT_LOG_ERR, + "pcre_exec() failed: %d on \"%*s\" using \"%V\"", + ret, length, subject, &re->pattern); + + return NXT_ERROR; + } + + return (ret != PCRE_ERROR_NOMATCH); +} diff --git a/src/nxt_pcre2.c b/src/nxt_pcre2.c new file mode 100644 index 00000000..22c4d2d4 --- /dev/null +++ b/src/nxt_pcre2.c @@ -0,0 +1,161 @@ +/* + * Copyright (C) Axel Duch + * Copyright (C) NGINX, Inc. + */ + +#include +#include + +#define PCRE2_CODE_UNIT_WIDTH 8 +#include + + +static void *nxt_pcre2_malloc(PCRE2_SIZE size, void *memory_data); +static void nxt_pcre2_free(void *p, void *memory_data); + + +struct nxt_regex_s { + pcre2_code *code; + nxt_str_t pattern; +}; + + +nxt_regex_t * +nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, nxt_regex_err_t *err) +{ + int errcode; + nxt_int_t ret; + PCRE2_SIZE erroffset; + nxt_regex_t *re; + pcre2_general_context *general_ctx; + pcre2_compile_context *compile_ctx; + + static const u_char alloc_error[] = "memory allocation failed"; + + general_ctx = pcre2_general_context_create(nxt_pcre2_malloc, + nxt_pcre2_free, mp); + if (nxt_slow_path(general_ctx == NULL)) { + goto alloc_fail; + } + + compile_ctx = pcre2_compile_context_create(general_ctx); + if (nxt_slow_path(compile_ctx == NULL)) { + goto alloc_fail; + } + + re = nxt_mp_get(mp, sizeof(nxt_regex_t)); + if (nxt_slow_path(re == NULL)) { + goto alloc_fail; + } + + if (nxt_slow_path(nxt_str_dup(mp, &re->pattern, source) == NULL)) { + goto alloc_fail; + } + + re->code = pcre2_compile((PCRE2_SPTR) source->start, source->length, 0, + &errcode, &erroffset, compile_ctx); + if (nxt_slow_path(re->code == NULL)) { + err->offset = erroffset; + + ret = pcre2_get_error_message(errcode, (PCRE2_UCHAR *) err->msg, + ERR_BUF_SIZE); + if (ret < 0) { + (void) nxt_sprintf(err->msg, err->msg + ERR_BUF_SIZE, + "compilation failed with unknown " + "error code: %d%Z", errcode); + } + + return NULL; + } + +#if 0 + errcode = pcre2_jit_compile(re, PCRE2_JIT_COMPLETE); + if (nxt_slow_path(errcode != 0 && errcode != PCRE2_ERROR_JIT_BADOPTION)) { + ret = pcre2_get_error_message(errcode, (PCRE2_UCHAR *) err->msg, + ERR_BUF_SIZE); + if (ret < 0) { + (void) nxt_sprintf(err->msg, err->msg + ERR_BUF_SIZE, + "JIT compilation failed with unknown " + "error code: %d%Z", errcode); + } + + return NULL; + } +#endif + + return re; + +alloc_fail: + + err->offset = source->length; + nxt_memcpy(err->msg, alloc_error, sizeof(alloc_error)); + + return NULL; +} + + +static void * +nxt_pcre2_malloc(PCRE2_SIZE size, void *mp) +{ + return nxt_mp_get(mp, size); +} + + +static void +nxt_pcre2_free(void *p, void *mp) +{ +} + + +nxt_regex_match_t * +nxt_regex_match_create(nxt_mp_t *mp, size_t size) +{ + nxt_regex_match_t *match; + pcre2_general_context *ctx; + + ctx = pcre2_general_context_create(nxt_pcre2_malloc, nxt_pcre2_free, mp); + if (nxt_slow_path(ctx == NULL)) { + nxt_thread_log_alert("pcre2_general_context_create() failed"); + return NULL; + } + + match = pcre2_match_data_create(size, ctx); + if (nxt_slow_path(match == NULL)) { + nxt_thread_log_alert("pcre2_match_data_create(%uz) failed", size); + return NULL; + } + + return match; +} + + +nxt_int_t +nxt_regex_match(nxt_regex_t *re, u_char *subject, size_t length, + nxt_regex_match_t *match) +{ + nxt_int_t ret; + PCRE2_UCHAR errptr[ERR_BUF_SIZE]; + + ret = pcre2_match(re->code, (PCRE2_SPTR) subject, length, 0, 0, match, + NULL); + + if (nxt_slow_path(ret < PCRE2_ERROR_NOMATCH)) { + + if (pcre2_get_error_message(ret, errptr, ERR_BUF_SIZE) < 0) { + nxt_thread_log_error(NXT_LOG_ERR, + "pcre2_match() failed: %d on \"%*s\" " + "using \"%V\"", ret, subject, length, subject, + &re->pattern); + + } else { + nxt_thread_log_error(NXT_LOG_ERR, + "pcre2_match() failed: %s (%d) on \"%*s\" " + "using \"%V\"", errptr, ret, length, subject, + &re->pattern); + } + + return NXT_ERROR; + } + + return (ret != PCRE2_ERROR_NOMATCH); +} diff --git a/src/nxt_regex.h b/src/nxt_regex.h new file mode 100644 index 00000000..a24c50f8 --- /dev/null +++ b/src/nxt_regex.h @@ -0,0 +1,41 @@ + +/* + * Copyright (C) Axel Duch + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_REGEX_H_INCLUDED_ +#define _NXT_REGEX_H_INCLUDED_ + +#if (NXT_HAVE_REGEX) + +typedef struct nxt_regex_s nxt_regex_t; + + #if (NXT_HAVE_PCRE2) +typedef void nxt_regex_match_t; +#else +typedef struct nxt_regex_match_s nxt_regex_match_t; +#endif + +typedef struct { + size_t offset; + +#if (NXT_HAVE_PCRE2) +#define ERR_BUF_SIZE 256 + u_char msg[ERR_BUF_SIZE]; +#else + const char *msg; +#endif +} nxt_regex_err_t; + + +NXT_EXPORT void nxt_regex_init(void); +NXT_EXPORT nxt_regex_t *nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, + nxt_regex_err_t *err); +NXT_EXPORT nxt_regex_match_t *nxt_regex_match_create(nxt_mp_t *mp, size_t size); +NXT_EXPORT nxt_int_t nxt_regex_match(nxt_regex_t *re, u_char *subject, + size_t length, nxt_regex_match_t *match); + +#endif /* NXT_HAVE_REGEX */ + +#endif /* _NXT_REGEX_H_INCLUDED_ */ diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index 44970b34..d9d986da 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -10,6 +10,7 @@ #include #include #include +#include static nxt_int_t nxt_runtime_inherited_listen_sockets(nxt_task_t *task, @@ -702,9 +703,6 @@ nxt_runtime_thread_pool_destroy(nxt_task_t *task, nxt_runtime_t *rt, static void nxt_runtime_thread_pool_init(void) { -#if (NXT_REGEX) - nxt_regex_init(0); -#endif } diff --git a/test/test_routing.py b/test/test_routing.py index 9e0f52dc..83852273 100644 --- a/test/test_routing.py +++ b/test/test_routing.py @@ -230,6 +230,48 @@ class TestRouting(TestApplicationProto): assert self.get(url='/aBCaBbc')['status'] == 200 assert self.get(url='/ABc')['status'] == 404 + def test_routes_empty_regex(self): + self.route_match({"uri":"~"}) + assert self.get(url='/')['status'] == 200, 'empty regexp' + assert self.get(url='/anything')['status'] == 200, '/anything' + + self.route_match({"uri":"!~"}) + assert self.get(url='/')['status'] == 404, 'empty regexp 2' + assert self.get(url='/nothing')['status'] == 404, '/nothing' + + def test_routes_bad_regex(self): + assert 'error' in self.route( + {"match": {"uri": "~/bl[ah"}, "action": {"return": 200}} + ), 'bad regex' + + status = self.route( + {"match": {"uri": "~(?R)?z"}, "action": {"return": 200}} + ) + if 'error' not in status: + assert self.get(url='/nothing_z')['status'] == 500, '/nothing_z' + + status = self.route( + {"match": {"uri": "~((?1)?z)"}, "action": {"return": 200}} + ) + if 'error' not in status: + assert self.get(url='/nothing_z')['status'] == 500, '/nothing_z' + + def test_routes_match_regex_case_sensitive(self): + self.route_match({"uri": "~/bl[ah]"}) + + assert self.get(url='/rlah')['status'] == 404, '/rlah' + assert self.get(url='/blah')['status'] == 200, '/blah' + assert self.get(url='/blh')['status'] == 200, '/blh' + assert self.get(url='/BLAH')['status'] == 404, '/BLAH' + + def test_routes_match_regex_negative_case_sensitive(self): + self.route_match({"uri": "!~/bl[ah]"}) + + assert self.get(url='/rlah')['status'] == 200, '/rlah' + assert self.get(url='/blah')['status'] == 404, '/blah' + assert self.get(url='/blh')['status'] == 404, '/blh' + assert self.get(url='/BLAH')['status'] == 200, '/BLAH' + def test_routes_pass_encode(self): def check_pass(path, name): assert 'success' in self.conf( -- cgit From fa1a3298ef777d921c84b15ea9ced2849d4b5115 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Thu, 19 Nov 2020 19:59:18 +0300 Subject: Added version 1.21.0 CHANGES. --- CHANGES | 46 ++++++++++++++++ docs/changes.xml | 164 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+) diff --git a/CHANGES b/CHANGES index 0373bb6f..b026fe5e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,50 @@ +Changes with Unit 1.21.0 19 Nov 2020 + + *) Change: procfs is mounted by default for all languages when "rootfs" + isolation is used. + + *) Change: any characters valid according to RFC 7230 are now allowed in + HTTP header field names. + + *) Change: HTTP header fields with underscores ("_") are now discarded + from requests by default. + + *) Feature: optional multithreaded request processing for Java, Python, + Perl, and Ruby apps. + + *) Feature: regular expressions in route matching patterns. + + *) Feature: compatibility with Python 3.9. + + *) Feature: the Python module now supports ASGI 2.0 legacy applications. + + *) Feature: the "protocol" option in Python applications aids choice + between ASGI and WSGI. + + *) Feature: the fastcgi_finish_request() PHP function that finalizes + request processing and continues code execution without holding onto + the client connection. + + *) Feature: the "discard_unsafe_fields" HTTP option that enables + discarding request header fields with irregular (but still valid) + characters in the field name. + + *) Feature: the "procfs" and "tmpfs" automount isolation options to + disable automatic mounting of eponymous filesystems. + + *) Bugfix: the router process could crash when running Go applications + under high load; the bug had appeared in 1.19.0. + + *) Bugfix: some language dependencies could remain mounted after using + "rootfs" isolation. + + *) Bugfix: various compatibility issues in Java applications. + + *) Bugfix: the Java module built with the musl C library couldn't run + applications that use "rootfs" isolation. + + Changes with Unit 1.20.0 08 Oct 2020 *) Change: the PHP module is now initialized before chrooting; this diff --git a/docs/changes.xml b/docs/changes.xml index aa413e2a..772d65f2 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -5,6 +5,170 @@ + + + + +Initial release of Java 15 module for NGINX Unit. + + + + + + + + + + +Initial release of Java 14 module for NGINX Unit. + + + + + + + + + + +Initial release of Java 13 module for NGINX Unit. + + + + + + + + + + +NGINX Unit updated to 1.21.0. + + + + + + + + + + +procfs is mounted by default for all languages when "rootfs" isolation is used. + + + + + +any characters valid according to RFC 7230 are now allowed in HTTP header field +names. + + + + + +HTTP header fields with underscores ("_") are now discarded from requests by +default. + + + + + +optional multithreaded request processing for Java, Python, Perl, and Ruby apps. + + + + + +regular expressions in route matching patterns. + + + + + +compatibility with Python 3.9. + + + + + +the Python module now supports ASGI 2.0 legacy applications. + + + + + +the "protocol" option in Python applications aids choice between ASGI and WSGI. + + + + + +the fastcgi_finish_request() PHP function that finalizes request processing and +continues code execution without holding onto the client connection. + + + + + +the "discard_unsafe_fields" HTTP option that enables discarding request header +fields with irregular (but still valid) characters in the field name. + + + + + +the "procfs" and "tmpfs" automount isolation options to disable automatic +mounting of eponymous filesystems. + + + + + +the router process could crash when running Go applications under high load; +the bug had appeared in 1.19.0. + + + + + +some language dependencies could remain mounted after using "rootfs" isolation. + + + + + +various compatibility issues in Java applications. + + + + + +the Java module built with the musl C library couldn't run applications that +use "rootfs" isolation. + + + + + + " -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.go1.11-dev b/pkg/docker/Dockerfile.go1.11-dev index 17b1d2e9..cb1c1d85 100644 --- a/pkg/docker/Dockerfile.go1.11-dev +++ b/pkg/docker/Dockerfile.go1.11-dev @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.jsc11 b/pkg/docker/Dockerfile.jsc11 index 65b8ce0f..f3d1d22d 100644 --- a/pkg/docker/Dockerfile.jsc11 +++ b/pkg/docker/Dockerfile.jsc11 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.minimal b/pkg/docker/Dockerfile.minimal index 5a10620a..da9c8712 100644 --- a/pkg/docker/Dockerfile.minimal +++ b/pkg/docker/Dockerfile.minimal @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.perl5.28 b/pkg/docker/Dockerfile.perl5.28 index 868527e5..976fa7b0 100644 --- a/pkg/docker/Dockerfile.perl5.28 +++ b/pkg/docker/Dockerfile.perl5.28 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.php7.3 b/pkg/docker/Dockerfile.php7.3 index b5789dec..f1178551 100644 --- a/pkg/docker/Dockerfile.php7.3 +++ b/pkg/docker/Dockerfile.php7.3 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.python2.7 b/pkg/docker/Dockerfile.python2.7 index 9bb83f7b..95241af6 100644 --- a/pkg/docker/Dockerfile.python2.7 +++ b/pkg/docker/Dockerfile.python2.7 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.python3.7 b/pkg/docker/Dockerfile.python3.7 index 99324844..52a5a257 100644 --- a/pkg/docker/Dockerfile.python3.7 +++ b/pkg/docker/Dockerfile.python3.7 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.ruby2.5 b/pkg/docker/Dockerfile.ruby2.5 index fef96867..342ccc9a 100644 --- a/pkg/docker/Dockerfile.ruby2.5 +++ b/pkg/docker/Dockerfile.ruby2.5 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ -- cgit From 806a9b2515c60b12a68cd97af04f7fa5cb4dffed Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Thu, 19 Nov 2020 21:13:19 +0300 Subject: Added tag 1.21.0 for changeset f804aaf7eee1 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 3d3b905b..89fff74d 100644 --- a/.hgtags +++ b/.hgtags @@ -27,3 +27,4 @@ b391df5f0102aa6afe660cfc863729c1b1111c9e 1.12.0 9e14c63773be52613dd47dea9fd113037f15a3eb 1.18.0 86cdf66f82745d8db35345368dcdb38c79a4f03a 1.19.0 0e985b30067380782125f1c479eda4ef909418df 1.20.0 +f804aaf7eee10a7d8116820840d6312dd4914a41 1.21.0 -- cgit