From 7a0383189ca8da1c227dad7a8258b0c4976e105e Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Wed, 19 Feb 2020 19:13:51 +0300 Subject: Version bump. --- version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version b/version index b694bfda..a0f9ae94 100644 --- a/version +++ b/version @@ -1,5 +1,5 @@ # Copyright (C) NGINX, Inc. -NXT_VERSION=1.15.0 -NXT_VERNUM=11500 +NXT_VERSION=1.16.0 +NXT_VERNUM=11600 -- cgit From 044b3afcdab3bc434f9340e97e83e37c5227be36 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Thu, 20 Feb 2020 17:58:24 +0300 Subject: Configuration: stripping comments from the input JSON. This allows to have JavaScript-like comments in the uploading JSON. --- src/nxt_conf.c | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/src/nxt_conf.c b/src/nxt_conf.c index 43820d2a..2a952c35 100644 --- a/src/nxt_conf.c +++ b/src/nxt_conf.c @@ -1244,15 +1244,74 @@ nxt_conf_json_parse(nxt_mp_t *mp, u_char *start, u_char *end, static u_char * nxt_conf_json_skip_space(u_char *start, u_char *end) { - u_char *p; + u_char *p, ch; + + enum { + sw_normal = 0, + sw_after_slash, + sw_single_comment, + sw_multi_comment, + sw_after_asterisk, + } state; + + state = sw_normal; for (p = start; nxt_fast_path(p != end); p++) { + ch = *p; + + switch (state) { + + case sw_normal: + switch (ch) { + case ' ': + case '\t': + case '\n': + case '\r': + continue; + case '/': + state = sw_after_slash; + continue; + } + + break; + + case sw_after_slash: + switch (ch) { + case '/': + state = sw_single_comment; + continue; + case '*': + state = sw_multi_comment; + continue; + } + + p--; + break; + + case sw_single_comment: + if (ch == '\n') { + state = sw_normal; + } - switch (*p) { - case ' ': - case '\t': - case '\r': - case '\n': + continue; + + case sw_multi_comment: + if (ch == '*') { + state = sw_after_asterisk; + } + + continue; + + case sw_after_asterisk: + switch (ch) { + case '/': + state = sw_normal; + continue; + case '*': + continue; + } + + state = sw_multi_comment; continue; } @@ -1346,6 +1405,7 @@ nxt_conf_json_parse_value(nxt_mp_t *mp, nxt_conf_value_t *value, u_char *start, case '{': case '[': case '"': + case '/': return p; } } -- cgit From d198a105eb9a49749fa38fe8eba4da59d572292e Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Thu, 20 Feb 2020 17:58:24 +0300 Subject: Configuration: removing UTF-8 BOM from the input JSON. Some editors can add it to JSON files. --- src/nxt_controller.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/nxt_controller.c b/src/nxt_controller.c index 86ba1246..cc1ed534 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -989,6 +989,13 @@ nxt_controller_process_config(nxt_task_t *task, nxt_controller_request_t *req, nxt_memzero(&error, sizeof(nxt_conf_json_error_t)); + /* Skip UTF-8 BOM. */ + if (nxt_buf_mem_used_size(mbuf) >= 3 + && nxt_memcmp(mbuf->pos, "\xEF\xBB\xBF", 3) == 0) + { + mbuf->pos += 3; + } + value = nxt_conf_json_parse(mp, mbuf->pos, mbuf->free, &error); if (value == NULL) { -- cgit From fcca366392ff42e1addfac958da70c3e5375fc35 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Thu, 20 Feb 2020 20:33:49 +0000 Subject: Tests: more tests with "max_body_size". --- test/test_settings.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/test_settings.py b/test/test_settings.py index 6b849558..9de3a928 100644 --- a/test/test_settings.py +++ b/test/test_settings.py @@ -215,6 +215,31 @@ Connection: close self.post(body='012345')['status'], 413, 'status size max' ) + def test_settings_max_body_size_large(self): + self.load('mirror') + + self.conf({'http': {'max_body_size': 32 * 1024 * 1024}}, 'settings') + + body = '0123456789abcdef' * 4 * 64 * 1024 + resp = self.post(body=body, read_buffer_size=1024 * 1024) + self.assertEqual(resp['status'], 200, 'status size 4') + self.assertEqual(resp['body'], body, 'status body 4') + + body = '0123456789abcdef' * 8 * 64 * 1024 + resp = self.post(body=body, read_buffer_size=1024 * 1024) + self.assertEqual(resp['status'], 200, 'status size 8') + self.assertEqual(resp['body'], body, 'status body 8') + + body = '0123456789abcdef' * 16 * 64 * 1024 + resp = self.post(body=body, read_buffer_size=1024 * 1024) + self.assertEqual(resp['status'], 200, 'status size 16') + self.assertEqual(resp['body'], body, 'status body 16') + + body = '0123456789abcdef' * 32 * 64 * 1024 + resp = self.post(body=body, read_buffer_size=1024 * 1024) + self.assertEqual(resp['status'], 200, 'status size 32') + self.assertEqual(resp['body'], body, 'status body 32') + @unittest.skip('not yet') def test_settings_negative_value(self): self.assertIn( -- cgit From 1f2445b01b04bf47409fe9aace990c7054a638a6 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Thu, 20 Feb 2020 21:06:31 +0000 Subject: Tests: added proxy test with large body. --- test/test_proxy.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/test_proxy.py b/test/test_proxy.py index 5d158285..74bd0873 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -188,6 +188,13 @@ Content-Length: 10 self.assertEqual(resp['status'], 200, 'status') self.assertEqual(resp['body'], payload, 'body') + self.conf({'http': {'max_body_size': 32 * 1024 * 1024}}, 'settings') + + payload = '0123456789abcdef' * 32 * 64 * 1024 + resp = self.post_http10(body=payload, read_buffer_size=1024 * 1024) + self.assertEqual(resp['status'], 200, 'status') + self.assertEqual(resp['body'], payload, 'body') + def test_proxy_parallel(self): payload = 'X' * 4096 * 257 buff_size = 4096 * 258 -- cgit From 5406d1e320eca0a94eeffe2e57e2e2b4d989e0d1 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Thu, 20 Feb 2020 21:06:56 +0000 Subject: Tests: added PHP test with invalid index extension only. --- test/test_php_application.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/test_php_application.py b/test/test_php_application.py index 837181e6..37606fa2 100644 --- a/test/test_php_application.py +++ b/test/test_php_application.py @@ -1,4 +1,6 @@ +import os import re +import shutil import unittest from unit.applications.lang.php import TestApplicationPHP @@ -512,5 +514,33 @@ class TestPHPApplication(TestApplicationPHP): self.get(url='/index.wrong')['status'], 200, 'status' ) + new_root = self.testdir + "/php" + os.mkdir(new_root) + shutil.copy(self.current_dir + '/php/phpinfo/index.wrong', new_root) + + self.assertIn( + 'success', + self.conf( + { + "listeners": {"*:7080": {"pass": "applications/phpinfo"}}, + "applications": { + "phpinfo": { + "type": "php", + "processes": {"spare": 0}, + "root": new_root, + "working_directory": new_root, + } + }, + } + ), + 'configure new root', + ) + + resp = self.get() + self.assertNotEqual( + str(resp['status']) + resp['body'], '200', 'status new root' + ) + + if __name__ == '__main__': TestPHPApplication.main() -- cgit From f519e31e2d97492d85aee13451554527540ccd90 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Fri, 21 Feb 2020 15:08:38 +0000 Subject: Tests: more static tests. --- test/test_static.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/test/test_static.py b/test/test_static.py index f9dcb7dd..b2489aa0 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -39,6 +39,9 @@ class TestStatic(TestApplicationProto): self.get(url='/index.html')['body'], '0123456789', 'index' ) self.assertEqual(self.get(url='/')['body'], '0123456789', 'index 2') + self.assertEqual(self.get(url='//')['body'], '0123456789', 'index 3') + self.assertEqual(self.get(url='/.')['body'], '0123456789', 'index 4') + self.assertEqual(self.get(url='/./')['body'], '0123456789', 'index 5') self.assertEqual( self.get(url='/?blah')['body'], '0123456789', 'index vars' ) @@ -199,10 +202,29 @@ class TestStatic(TestApplicationProto): self.get(url='/link/file')['status'], 200, 'symlink file' ) - def test_static_head(self): - resp = self.head(url='/') - self.assertEqual(resp['status'], 200, 'status') - self.assertEqual(resp['body'], '', 'empty body') + def test_static_method(self): + resp = self.head() + self.assertEqual(resp['status'], 200, 'HEAD status') + self.assertEqual(resp['body'], '', 'HEAD empty body') + + self.assertEqual(self.delete()['status'], 405, 'DELETE') + self.assertEqual(self.post()['status'], 405, 'POST') + self.assertEqual(self.put()['status'], 405, 'PUT') + + def test_static_path(self): + self.assertEqual( + self.get(url='/dir/../dir/file')['status'], 200, 'relative' + ) + + self.assertEqual(self.get(url='./')['status'], 400, 'path invalid') + self.assertEqual(self.get(url='../')['status'], 400, 'path invalid 2') + self.assertEqual(self.get(url='/..')['status'], 400, 'path invalid 3') + self.assertEqual( + self.get(url='../assets/')['status'], 400, 'path invalid 4' + ) + self.assertEqual( + self.get(url='/../assets/')['status'], 400, 'path invalid 5' + ) def test_static_two_clients(self): _, sock = self.get(url='/', start=True, no_recv=True) -- cgit From 98c0ce6cc413b174d5a36f92095246a9967b1371 Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Tue, 25 Feb 2020 15:55:31 +0000 Subject: PHP: fixed php >= 7.4 with zts enabled. --- src/nxt_php_sapi.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 26bf915f..917bc1c9 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -15,21 +15,19 @@ #include -#if PHP_MAJOR_VERSION >= 7 -# define NXT_PHP7 1 -# if PHP_MINOR_VERSION >= 1 -# define NXT_HAVE_PHP_LOG_MESSAGE_WITH_SYSLOG_TYPE 1 -# else -# define NXT_HAVE_PHP_INTERRUPTS 1 -# endif -# define NXT_HAVE_PHP_IGNORE_CWD 1 +#if PHP_VERSION_ID >= 50400 +#define NXT_HAVE_PHP_IGNORE_CWD 1 +#endif + +#if PHP_VERSION_ID >= 70100 +#define NXT_HAVE_PHP_LOG_MESSAGE_WITH_SYSLOG_TYPE 1 #else -# define NXT_HAVE_PHP_INTERRUPTS 1 -# if PHP_MINOR_VERSION >= 4 -# define NXT_HAVE_PHP_IGNORE_CWD 1 -# endif +#define NXT_HAVE_PHP_INTERRUPTS 1 #endif +#if PHP_VERSION_ID >= 70000 +#define NXT_PHP7 1 +#endif typedef struct nxt_php_run_ctx_s nxt_php_run_ctx_t; @@ -172,7 +170,7 @@ NXT_EXPORT nxt_app_module_t nxt_app_module = { static nxt_task_t *nxt_php_task; -#ifdef ZTS +#if defined(ZTS) && PHP_VERSION_ID < 70400 static void ***tsrm_ls; #endif @@ -278,10 +276,16 @@ nxt_php_init(nxt_task_t *task, nxt_common_app_conf_t *conf) } #ifdef ZTS + +#if PHP_VERSION_ID >= 70400 + php_tsrm_startup(); +#else tsrm_startup(1, 1, 0, NULL); tsrm_ls = ts_resource(0); #endif +#endif + #if defined(NXT_PHP7) && defined(ZEND_SIGNALS) #if (NXT_ZEND_SIGNAL_STARTUP) -- cgit From fbc72d7fec338466ff65ab30fc8d51e963f98d70 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Thu, 27 Feb 2020 01:37:54 +0000 Subject: Tests: added test with invalid IPv6 address in routing block. --- test/test_routing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_routing.py b/test/test_routing.py index eb7b2fd8..67f10d10 100644 --- a/test/test_routing.py +++ b/test/test_routing.py @@ -1690,6 +1690,7 @@ class TestRouting(TestApplicationProto): self.route_match_invalid({"source": "2001::/129"}) self.route_match_invalid({"source": "::FFFFF"}) self.route_match_invalid({"source": "[::1]:"}) + self.route_match_invalid({"source": "[:::]:7080"}) self.route_match_invalid({"source": "*:"}) self.route_match_invalid({"source": "*:1-a"}) self.route_match_invalid({"source": "*:65536"}) -- cgit From 5d67879a5051f7e3d3b4dd96da4f43e7822f7da4 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Thu, 27 Feb 2020 18:41:24 +0000 Subject: Tests: added "-r" option to print unit.log on failures. --- test/unit/main.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/test/unit/main.py b/test/unit/main.py index 37d01d3b..149cfb2c 100644 --- a/test/unit/main.py +++ b/test/unit/main.py @@ -30,6 +30,7 @@ class TestUnit(unittest.TestCase): detailed = False save_log = False + print_log = False unsafe = False def __init__(self, methodName='runTest'): @@ -183,7 +184,7 @@ class TestUnit(unittest.TestCase): shutil.rmtree(self.testdir) else: - self._print_path_to_log() + self._print_log() def stop(self): if self._started: @@ -281,14 +282,14 @@ class TestUnit(unittest.TestCase): alerts = [al for al in alerts if re.search(skip, al) is None] if alerts: - self._print_path_to_log() + self._print_log(log) self.assertFalse(alerts, 'alert(s)') if not self.skip_sanitizer: sanitizer_errors = re.findall('.+Sanitizer.+', log) if sanitizer_errors: - self._print_path_to_log() + self._print_log(log) self.assertFalse(sanitizer_errors, 'sanitizer error(s)') if found: @@ -360,6 +361,13 @@ class TestUnit(unittest.TestCase): action='store_true', help='Save unit.log after the test execution', ) + parser.add_argument( + '-r', + '--reprint_log', + dest='print_log', + action='store_true', + help='Print unit.log to stdout in case of errors', + ) parser.add_argument( '-u', '--unsafe', @@ -374,6 +382,7 @@ class TestUnit(unittest.TestCase): def _set_args(args): TestUnit.detailed = args.detailed TestUnit.save_log = args.save_log + TestUnit.print_log = args.print_log TestUnit.unsafe = args.unsafe # set stdout to non-blocking @@ -381,5 +390,15 @@ class TestUnit(unittest.TestCase): if TestUnit.detailed: fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, 0) - def _print_path_to_log(self): - print('Path to unit.log:\n' + self.testdir + '/unit.log') + def _print_log(self, data=None): + path = self.testdir + '/unit.log' + + print('Path to unit.log:\n' + path + '\n') + + if TestUnit.print_log: + if data is None: + with open(path, 'r', encoding='utf-8', errors='ignore') as f: + data = f.read() + + print(data) + -- cgit From f68947bc60c930d460e1bd71bb5f486579009578 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Mon, 2 Mar 2020 13:10:38 +0000 Subject: Tests: truncated huge messages while logging. --- test/unit/http.py | 45 +++++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/test/unit/http.py b/test/unit/http.py index c71e8f7e..47fb48f1 100644 --- a/test/unit/http.py +++ b/test/unit/http.py @@ -96,12 +96,7 @@ class TestHTTP(TestUnit): encoding = 'utf-8' if 'encoding' not in kwargs else kwargs['encoding'] - if TestUnit.detailed: - print('>>>') - try: - print(req.decode(encoding, 'ignore')) - except UnicodeEncodeError: - print(req) + self.log_out(req, encoding) resp = '' @@ -113,12 +108,7 @@ class TestHTTP(TestUnit): sock, read_timeout=read_timeout, buff_size=read_buffer_size ).decode(encoding) - if TestUnit.detailed: - print('<<<') - try: - print(resp) - except UnicodeEncodeError: - print(resp.encode()) + self.log_in(resp) if 'raw_resp' not in kwargs: resp = self._resp_to_dict(resp) @@ -138,6 +128,37 @@ class TestHTTP(TestUnit): return (resp, sock) + def log_out(self, log, encoding): + if TestUnit.detailed: + print('>>>') + log = self.log_truncate(log) + try: + print(log.decode(encoding, 'ignore')) + except UnicodeEncodeError: + print(log) + + def log_in(self, log): + if TestUnit.detailed: + print('<<<') + log = self.log_truncate(log) + try: + print(log) + except UnicodeEncodeError: + print(log.encode()) + + def log_truncate(self, log, limit=1024): + len_log = len(log) + if len_log > limit: + log = log[:limit] + appendix = '(...logged %s of %s bytes)' % (limit, len_log) + + if isinstance(log, bytes): + appendix = appendix.encode() + + log = log + appendix + + return log + def delete(self, **kwargs): return self.http('DELETE', **kwargs) -- cgit From 004ab48a9ee88c3e9a7225c0c83910329ac01265 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Tue, 3 Mar 2020 18:28:16 +0300 Subject: Node.js: fixing x86 warning about the signed/unsigned comparison. --- src/nodejs/unit-http/unit.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nodejs/unit-http/unit.cpp b/src/nodejs/unit-http/unit.cpp index 64e076c1..975174d4 100644 --- a/src/nodejs/unit-http/unit.cpp +++ b/src/nodejs/unit-http/unit.cpp @@ -812,8 +812,7 @@ Unit::response_write(napi_env env, napi_callback_info info) /* TODO: will work only for utf8 content-type */ if (req->response_buf != NULL - && (req->response_buf->end - req->response_buf->free) - >= buf_len) + && req->response_buf->end >= req->response_buf->free + buf_len) { buf = req->response_buf; -- cgit From c74f3a6c56a3d15c0898d0336986c83f3e820bbc Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Tue, 3 Mar 2020 18:28:20 +0300 Subject: Java: fixing Spring applications start. This closes #403 issue on GitHub. --- src/java/nginx/unit/Context.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/nginx/unit/Context.java b/src/java/nginx/unit/Context.java index 6fcd6018..5f7ec22f 100644 --- a/src/java/nginx/unit/Context.java +++ b/src/java/nginx/unit/Context.java @@ -1517,7 +1517,7 @@ public class Context implements ServletContext, InitParams || ci.isAnnotation() || ci.isAbstract()) { - return; + continue; } trace("loadInitializer: handles class: " + ci.getName()); -- cgit From 9e295fa3141e8deec7813ea4e0c6fa57d4a87bd8 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Tue, 3 Mar 2020 18:28:26 +0300 Subject: Fixing request_app_link reference counting for delayed requests. Router built with debug may stop with assertion during stalled requests re-schedule. This was caused by missing reference counting increment before nxt_router_port_select() call. --- src/nxt_router.c | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 3ff048c5..6ef4ee78 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -4152,6 +4152,13 @@ nxt_router_app_port_release(nxt_task_t *task, nxt_port_t *port, state.req_app_link = re_ra; state.app = app; + /* + * Need to increment use count "in advance" because + * nxt_router_port_select() will remove re_ra from lists + * and decrement use count. + */ + nxt_request_app_link_inc_use(re_ra); + nxt_router_port_select(task, &state); goto re_ra_cancelled; @@ -4217,16 +4224,18 @@ re_ra_cancelled: if (re_ra != NULL) { if (nxt_router_port_post_select(task, &state) == NXT_OK) { /* - * There should be call nxt_request_app_link_inc_use(re_ra), - * because of one more link in the queue. - * Corresponding decrement is in nxt_router_app_process_request(). + * Reference counter already incremented above, this will + * keep re_ra while nxt_router_app_process_request() + * task is in queue. Reference counter decreased in + * nxt_router_app_process_request() after processing. */ - nxt_request_app_link_inc_use(re_ra); - nxt_work_queue_add(&task->thread->engine->fast_work_queue, nxt_router_app_process_request, &task->thread->engine->task, app, re_ra); + + } else { + nxt_request_app_link_use(task, re_ra, -1); } } @@ -4234,15 +4243,14 @@ re_ra_cancelled: /* * There should be call nxt_request_app_link_inc_use(req_app_link), * because of one more link in the queue. But one link was - * recently removed from app->requests link. + * recently removed from app->requests linked list. + * Corresponding decrement is in nxt_router_app_process_request(). */ nxt_work_queue_add(&task->thread->engine->fast_work_queue, nxt_router_app_process_request, &task->thread->engine->task, app, req_app_link); - /* ... skip nxt_request_app_link_use(task, req_app_link, -1) too. */ - goto adjust_use; } @@ -5185,6 +5193,13 @@ nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data) state.req_app_link = pending_ra; state.app = app; + /* + * Need to increment use count "in advance" because + * nxt_router_port_select() will remove pending_ra from lists + * and decrement use count. + */ + nxt_request_app_link_inc_use(pending_ra); + nxt_router_port_select(task, &state); } else { @@ -5196,7 +5211,19 @@ nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data) if (pending_ra != NULL) { if (nxt_router_port_post_select(task, &state) == NXT_OK) { - nxt_router_app_prepare_request(task, pending_ra); + /* + * Reference counter already incremented above, this will + * keep pending_ra while nxt_router_app_process_request() + * task is in queue. Reference counter decreased in + * nxt_router_app_process_request() after processing. + */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + nxt_router_app_process_request, + &task->thread->engine->task, app, pending_ra); + + } else { + nxt_request_app_link_use(task, pending_ra, -1); } } -- cgit From f99d20ad39a62cf30b6b0b01593336572484f4f5 Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Tue, 3 Mar 2020 14:38:08 +0000 Subject: PHP: optimization to avoid surplus chdir(2) calls. For each request, the worker calls the php_execute_script function from libphp that changes to the script directory before doing its work and then restores the process directory before returning. The chdir(2) calls it performs are unnecessary in Unit design. In simple benchmarks, profiling shows that the chdir syscall code path (syscall, FS walk, etc.) is where the CPU spends most of its time. PHP SAPI semantics requires the script to be run from the script directory. In Unit's PHP implementation, we have two use cases: - script - arbitrary path The "script" configuration doesn't have much need for a working directory change: it can be changed once at module initialization. The module needs to chdir again only if the user's PHP script also calls chdir to switch to another directory during execution. If "script" is not used in Unit configuration, we must ensure the script is run from its directory (thus calling chdir before exec), but there's no need to restore the working directory later. Our implementation disables mandatory chdir calls with the SAPI option SAPI_OPTION_NO_CHDIR, instead calling chdir only when needed. To detect the user's calls to chdir, a simple "unit" extension is added that hooks the built-in chdir() PHP call. --- auto/modules/php | 13 +- src/nxt_php_sapi.c | 398 ++++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 310 insertions(+), 101 deletions(-) diff --git a/auto/modules/php b/auto/modules/php index 97d1ac43..51b068e7 100644 --- a/auto/modules/php +++ b/auto/modules/php @@ -58,6 +58,7 @@ NXT_PHP=${NXT_PHP_CONFIG%-config*} NXT_PHP_MODULE=${NXT_PHP_MODULE=${NXT_PHP##*/}} NXT_PHP_LIB_PATH=${NXT_PHP_LIB_PATH=} NXT_PHP_LIB_STATIC=${NXT_PHP_LIB_STATIC=no} +NXT_PHP_ADDITIONAL_FLAGS= $echo "configuring PHP module" @@ -75,6 +76,14 @@ if /bin/sh -c "${NXT_PHP_CONFIG} --version" >> $NXT_AUTOCONF_ERR 2>&1; then NXT_PHP_VERSION="`${NXT_PHP_CONFIG} --version`" $echo " + PHP SAPI: [`${NXT_PHP_CONFIG} --php-sapis`]" + NXT_PHP_MAJOR_VERSION=${NXT_PHP_VERSION%%.*} + NXT_PHP_MINOR_VERSION=${NXT_PHP_VERSION#??} + NXT_PHP_MINOR_VERSION=${NXT_PHP_MINOR_VERSION%.*} + + if [ $NXT_PHP_MAJOR_VERSION = 5 -a $NXT_PHP_MINOR_VERSION -lt 4 ]; then + NXT_PHP_ADDITIONAL_FLAGS=-Wno-write-strings + fi + NXT_PHP_INCLUDE="`${NXT_PHP_CONFIG} --includes`" if [ $NXT_PHP_LIB_STATIC = yes ]; then @@ -204,8 +213,8 @@ for nxt_src in $NXT_PHP_MODULE_SRCS; do cat << END >> $NXT_MAKEFILE $NXT_BUILD_DIR/$nxt_obj: $nxt_src $NXT_VERSION_H - \$(CC) -c \$(CFLAGS) \$(NXT_INCS) $NXT_PHP_INCLUDE \\ - -DNXT_ZEND_SIGNAL_STARTUP=$NXT_ZEND_SIGNAL_STARTUP \\ + \$(CC) -c \$(CFLAGS) $NXT_PHP_ADDITIONAL_FLAGS \$(NXT_INCS) \\ + $NXT_PHP_INCLUDE -DNXT_ZEND_SIGNAL_STARTUP=$NXT_ZEND_SIGNAL_STARTUP \\ $nxt_dep_flags \\ -o $NXT_BUILD_DIR/$nxt_obj $nxt_src $nxt_dep_post diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 917bc1c9..3702c371 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -29,7 +29,17 @@ #define NXT_PHP7 1 #endif -typedef struct nxt_php_run_ctx_s nxt_php_run_ctx_t; +typedef struct { + char *cookie; + nxt_str_t path_info; + nxt_str_t script_name; + nxt_str_t script_filename; + nxt_str_t script_dirname; + nxt_unit_request_info_t *req; + + uint8_t chdir; /* 1 bit */ +} nxt_php_run_ctx_t; + #ifdef NXT_PHP7 typedef int (*nxt_php_disable_t)(char *p, size_t size); @@ -37,14 +47,24 @@ typedef int (*nxt_php_disable_t)(char *p, size_t size); typedef int (*nxt_php_disable_t)(char *p, uint TSRMLS_DC); #endif +#if PHP_VERSION_ID < 70200 +typedef void (*zif_handler)(INTERNAL_FUNCTION_PARAMETERS); +#endif + static nxt_int_t nxt_php_init(nxt_task_t *task, nxt_common_app_conf_t *conf); static void nxt_php_str_trim_trail(nxt_str_t *str, u_char t); static void nxt_php_str_trim_lead(nxt_str_t *str, u_char t); +static nxt_int_t nxt_php_dirname(const nxt_str_t *file, nxt_str_t *dir); nxt_inline u_char *nxt_realpath(const void *c); +nxt_inline void nxt_php_vcwd_chdir(nxt_unit_request_info_t *req, + const nxt_str_t *dirname); -static void nxt_php_request_handler(nxt_unit_request_info_t *req); +static void nxt_php_script_request_handler(nxt_unit_request_info_t *req); +static void nxt_php_path_request_handler(nxt_unit_request_info_t *req); +static nxt_int_t nxt_php_request_init(nxt_php_run_ctx_t *ctx, + nxt_unit_request_t *r); static int nxt_php_startup(sapi_module_struct *sapi_module); static void nxt_php_set_options(nxt_task_t *task, nxt_conf_value_t *options, @@ -54,6 +74,8 @@ static nxt_int_t nxt_php_alter_option(nxt_str_t *name, nxt_str_t *value, static void nxt_php_disable(nxt_task_t *task, const char *type, nxt_str_t *value, char **ptr, nxt_php_disable_t disable); static int nxt_php_send_headers(sapi_headers_struct *sapi_headers TSRMLS_DC); +static void *nxt_php_hash_str_find_ptr(const HashTable *ht, + const nxt_str_t *str); static char *nxt_php_read_cookies(TSRMLS_D); static void nxt_php_set_sptr(nxt_unit_request_info_t *req, const char *name, nxt_unit_sptr_t *v, uint32_t len, zval *track_vars_array TSRMLS_DC); @@ -78,6 +100,55 @@ static int nxt_php_read_post(char *buffer, uint count_bytes TSRMLS_DC); #endif +PHP_MINIT_FUNCTION(nxt_php_ext); +ZEND_NAMED_FUNCTION(nxt_php_chdir); + +zif_handler nxt_php_chdir_handler; + + +static zend_module_entry nxt_php_unit_module = { + STANDARD_MODULE_HEADER, + "unit", + NULL, /* function table */ + PHP_MINIT(nxt_php_ext), /* initialization */ + NULL, /* shutdown */ + NULL, /* request initialization */ + NULL, /* request shutdown */ + NULL, /* information */ + NXT_VERSION, + STANDARD_MODULE_PROPERTIES +}; + + +PHP_MINIT_FUNCTION(nxt_php_ext) +{ + zend_function *func; + + static const nxt_str_t chdir = nxt_string("chdir"); + + func = nxt_php_hash_str_find_ptr(CG(function_table), &chdir); + if (nxt_slow_path(func == NULL)) { + return FAILURE; + } + + nxt_php_chdir_handler = func->internal_function.handler; + func->internal_function.handler = nxt_php_chdir; + + return SUCCESS; +} + + +ZEND_NAMED_FUNCTION(nxt_php_chdir) +{ + nxt_php_run_ctx_t *ctx; + + ctx = SG(server_context); + ctx->chdir = 1; + + nxt_php_chdir_handler(INTERNAL_FUNCTION_PARAM_PASSTHRU); +} + + static sapi_module_struct nxt_php_sapi_module = { (char *) "cli-server", @@ -139,17 +210,9 @@ static sapi_module_struct nxt_php_sapi_module = }; -struct nxt_php_run_ctx_s { - char *cookie; - nxt_str_t path_info; - nxt_str_t script_name; - nxt_str_t script_filename; - nxt_unit_request_info_t *req; -}; - - static nxt_str_t nxt_php_root; static nxt_str_t nxt_php_script_name; +static nxt_str_t nxt_php_script_dirname; static nxt_str_t nxt_php_script_filename; static nxt_str_t nxt_php_index = nxt_string("index.php"); @@ -180,7 +243,9 @@ nxt_php_init(nxt_task_t *task, nxt_common_app_conf_t *conf) { u_char *p, *tmp; nxt_str_t ini_path; - nxt_str_t *root, *script_filename, *script_name, *index; + nxt_str_t *root, *script_filename, *script_dirname, *script_name; + nxt_str_t *index; + nxt_int_t ret; nxt_port_t *my_port, *main_port; nxt_runtime_t *rt; nxt_unit_ctx_t *unit_ctx; @@ -203,6 +268,7 @@ nxt_php_init(nxt_task_t *task, nxt_common_app_conf_t *conf) root = &nxt_php_root; script_filename = &nxt_php_script_filename; + script_dirname = &nxt_php_script_dirname; script_name = &nxt_php_script_name; index = &nxt_php_index; @@ -247,6 +313,11 @@ nxt_php_init(nxt_task_t *task, nxt_common_app_conf_t *conf) return NXT_ERROR; } + ret = nxt_php_dirname(script_filename, script_dirname); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + script_name->length = c->script.length + 1; script_name->start = nxt_malloc(script_name->length); if (nxt_slow_path(script_name->start == NULL)) { @@ -275,6 +346,22 @@ nxt_php_init(nxt_task_t *task, nxt_common_app_conf_t *conf) nxt_memcpy(index->start, c->index.start, c->index.length); } + nxt_memzero(&php_init, sizeof(nxt_unit_init_t)); + + if (nxt_php_script_filename.start != NULL) { + if (nxt_slow_path(chdir((char *) script_dirname->start) != 0)) { + nxt_alert(task, "failed to chdir(%V) %E", script_dirname, + nxt_errno); + + return NXT_ERROR; + } + + php_init.callbacks.request_handler = nxt_php_script_request_handler; + + } else { + php_init.callbacks.request_handler = nxt_php_path_request_handler; + } + #ifdef ZTS #if PHP_VERSION_ID >= 70400 @@ -316,7 +403,10 @@ nxt_php_init(nxt_task_t *task, nxt_common_app_conf_t *conf) } } - nxt_php_startup(&nxt_php_sapi_module); + if (nxt_slow_path(nxt_php_startup(&nxt_php_sapi_module) == FAILURE)) { + nxt_alert(task, "failed to initialize SAPI module and extension"); + return NXT_ERROR; + } if (c->options != NULL) { value = nxt_conf_get_object_member(c->options, &admin_str, NULL); @@ -326,21 +416,20 @@ nxt_php_init(nxt_task_t *task, nxt_common_app_conf_t *conf) nxt_php_set_options(task, value, ZEND_INI_USER); } - nxt_memzero(&php_init, sizeof(nxt_unit_init_t)); - rt = task->thread->runtime; main_port = rt->port_by_type[NXT_PROCESS_MAIN]; if (nxt_slow_path(main_port == NULL)) { + nxt_alert(task, "main process not found"); return NXT_ERROR; } my_port = nxt_runtime_port_find(rt, nxt_pid, 0); if (nxt_slow_path(my_port == NULL)) { + nxt_alert(task, "my_port not found"); return NXT_ERROR; } - php_init.callbacks.request_handler = nxt_php_request_handler; php_init.ready_port.id.pid = main_port->pid; php_init.ready_port.id.id = main_port->id; php_init.ready_port.out_fd = main_port->pair[1]; @@ -415,7 +504,7 @@ nxt_php_set_options(nxt_task_t *task, nxt_conf_value_t *options, int type) } -#if (NXT_PHP7) +#ifdef NXT_PHP7 static nxt_int_t nxt_php_alter_option(nxt_str_t *name, nxt_str_t *value, int type) @@ -423,10 +512,8 @@ nxt_php_alter_option(nxt_str_t *name, nxt_str_t *value, int type) zend_string *zs; zend_ini_entry *ini_entry; - ini_entry = zend_hash_str_find_ptr(EG(ini_directives), - (char *) name->start, name->length); - - if (ini_entry == NULL) { + ini_entry = nxt_php_hash_str_find_ptr(EG(ini_directives), name); + if (nxt_slow_path(ini_entry == NULL)) { return NXT_ERROR; } @@ -456,19 +543,9 @@ nxt_php_alter_option(nxt_str_t *name, nxt_str_t *value, int type) { char *cstr; zend_ini_entry *ini_entry; - char buf[256]; - if (nxt_slow_path(name->length >= sizeof(buf))) { - return NXT_ERROR; - } - - nxt_memcpy(buf, name->start, name->length); - buf[name->length] = '\0'; - - if (zend_hash_find(EG(ini_directives), buf, name->length + 1, - (void **) &ini_entry) - == FAILURE) - { + ini_entry = nxt_php_hash_str_find_ptr(EG(ini_directives), name); + if (nxt_slow_path(ini_entry == NULL)) { return NXT_ERROR; } @@ -553,6 +630,33 @@ nxt_php_disable(nxt_task_t *task, const char *type, nxt_str_t *value, } +static nxt_int_t +nxt_php_dirname(const nxt_str_t *file, nxt_str_t *dir) +{ + size_t length; + + nxt_assert(file->length > 0 && file->start[0] == '/'); + + length = file->length; + + while (file->start[length - 1] != '/') { + length--; + } + + dir->length = length; + dir->start = nxt_malloc(length + 1); + if (nxt_slow_path(dir->start == NULL)) { + return NXT_ERROR; + } + + nxt_memcpy(dir->start, file->start, length); + + dir->start[length] = '\0'; + + return NXT_OK; +} + + static void nxt_php_str_trim_trail(nxt_str_t *str, u_char t) { @@ -582,12 +686,46 @@ nxt_realpath(const void *c) static void -nxt_php_request_handler(nxt_unit_request_info_t *req) +nxt_php_script_request_handler(nxt_unit_request_info_t *req) +{ + zend_file_handle file_handle; + nxt_php_run_ctx_t ctx; + + nxt_memzero(&ctx, sizeof(ctx)); + + ctx.req = req; + ctx.script_filename = nxt_php_script_filename; + ctx.script_dirname = nxt_php_script_dirname; + ctx.script_name = nxt_php_script_name; + + nxt_memzero(&file_handle, sizeof(file_handle)); + + file_handle.type = ZEND_HANDLE_FILENAME; + file_handle.filename = (char *) ctx.script_filename.start; + + if (nxt_slow_path(nxt_php_request_init(&ctx, req->request) != NXT_OK)) { + nxt_unit_request_done(req, NXT_UNIT_ERROR); + return; + } + + php_execute_script(&file_handle TSRMLS_CC); + + if (ctx.chdir) { + nxt_php_vcwd_chdir(ctx.req, &nxt_php_script_dirname); + } + + php_request_shutdown(NULL); + + nxt_unit_request_done(req, NXT_UNIT_OK); +} + + +static void +nxt_php_path_request_handler(nxt_unit_request_info_t *req) { - int rc; u_char *p; nxt_str_t path, script_name; - nxt_unit_field_t *f; + nxt_int_t ret; zend_file_handle file_handle; nxt_php_run_ctx_t run_ctx, *ctx; nxt_unit_request_t *r; @@ -602,59 +740,101 @@ nxt_php_request_handler(nxt_unit_request_info_t *req) path.length = r->path_length; path.start = nxt_unit_sptr_get(&r->path); - if (nxt_php_script_filename.start == NULL) { - nxt_str_null(&script_name); - - ctx->path_info.start = (u_char *) strstr((char *) path.start, ".php/"); - if (ctx->path_info.start != NULL) { - ctx->path_info.start += 4; - path.length = ctx->path_info.start - path.start; + nxt_str_null(&script_name); - ctx->path_info.length = r->path_length - path.length; + ctx->path_info.start = (u_char *) strstr((char *) path.start, ".php/"); + if (ctx->path_info.start != NULL) { + ctx->path_info.start += 4; + path.length = ctx->path_info.start - path.start; - } else if (path.start[path.length - 1] == '/') { - script_name = nxt_php_index; + ctx->path_info.length = r->path_length - path.length; - } else { - if (nxt_slow_path(path.length < 4 - || nxt_memcmp(path.start + (path.length - 4), - ".php", 4))) - { - nxt_unit_request_done(req, NXT_UNIT_ERROR); + } else if (path.start[path.length - 1] == '/') { + script_name = nxt_php_index; - return; - } - } - - ctx->script_filename.length = nxt_php_root.length + path.length - + script_name.length; - p = nxt_malloc(ctx->script_filename.length + 1); - if (nxt_slow_path(p == NULL)) { + } else { + if (nxt_slow_path(path.length < 4 + || nxt_memcmp(path.start + (path.length - 4), + ".php", 4))) + { nxt_unit_request_done(req, NXT_UNIT_ERROR); return; } + } + + ctx->script_filename.length = nxt_php_root.length + + path.length + + script_name.length; - ctx->script_filename.start = p; + p = nxt_malloc(ctx->script_filename.length + 1); + if (nxt_slow_path(p == NULL)) { + nxt_unit_request_done(req, NXT_UNIT_ERROR); - ctx->script_name.length = path.length + script_name.length; - ctx->script_name.start = p + nxt_php_root.length; + return; + } - p = nxt_cpymem(p, nxt_php_root.start, nxt_php_root.length); - p = nxt_cpymem(p, path.start, path.length); + ctx->script_filename.start = p; - if (script_name.length > 0) { - p = nxt_cpymem(p, script_name.start, script_name.length); - } + ctx->script_name.length = path.length + script_name.length; + ctx->script_name.start = p + nxt_php_root.length; - *p = '\0'; + p = nxt_cpymem(p, nxt_php_root.start, nxt_php_root.length); + p = nxt_cpymem(p, path.start, path.length); - } else { - ctx->script_filename = nxt_php_script_filename; - ctx->script_name = nxt_php_script_name; + if (script_name.length > 0) { + p = nxt_cpymem(p, script_name.start, script_name.length); + } + + *p = '\0'; + + nxt_memzero(&file_handle, sizeof(file_handle)); + + file_handle.type = ZEND_HANDLE_FILENAME; + file_handle.filename = (char *) ctx->script_filename.start; + + ret = nxt_php_dirname(&ctx->script_filename, &ctx->script_dirname); + if (nxt_slow_path(ret != NXT_OK)) { + nxt_unit_request_done(req, NXT_UNIT_ERROR); + nxt_free(ctx->script_filename.start); + + return; } + if (nxt_slow_path(nxt_php_request_init(ctx, req->request) != NXT_OK)) { + nxt_unit_request_done(req, NXT_UNIT_ERROR); + goto cleanup; + } + + nxt_php_vcwd_chdir(ctx->req, &ctx->script_dirname); + + php_execute_script(&file_handle TSRMLS_CC); + + php_request_shutdown(NULL); + + nxt_unit_request_done(req, NXT_UNIT_OK); + +cleanup: + + nxt_free(ctx->script_filename.start); + nxt_free(ctx->script_dirname.start); +} + + +static int +nxt_php_startup(sapi_module_struct *sapi_module) +{ + return php_module_startup(sapi_module, &nxt_php_unit_module, 1); +} + + +static nxt_int_t +nxt_php_request_init(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) +{ + nxt_unit_field_t *f; + SG(server_context) = ctx; + SG(options) |= SAPI_OPTION_NO_CHDIR; SG(request_info).request_uri = nxt_unit_sptr_get(&r->target); SG(request_info).request_method = nxt_unit_sptr_get(&r->method); @@ -680,55 +860,40 @@ nxt_php_request_handler(nxt_unit_request_info_t *req) SG(request_info).path_translated = NULL; - nxt_memzero(&file_handle, sizeof(file_handle)); - - file_handle.type = ZEND_HANDLE_FILENAME; - file_handle.filename = (char *) ctx->script_filename.start; - - nxt_unit_req_debug(req, "handle.filename = '%s'", + nxt_unit_req_debug(ctx->req, "handle.filename = '%s'", ctx->script_filename.start); if (nxt_php_script_filename.start != NULL) { - nxt_unit_req_debug(req, "run script %.*s in absolute mode", + nxt_unit_req_debug(ctx->req, "run script %.*s in absolute mode", (int) nxt_php_script_filename.length, (char *) nxt_php_script_filename.start); } else { - nxt_unit_req_debug(req, "run script %.*s", + nxt_unit_req_debug(ctx->req, "run script %.*s", (int) ctx->script_filename.length, (char *) ctx->script_filename.start); } -#if (NXT_PHP7) +#ifdef NXT_PHP7 if (nxt_slow_path(php_request_startup() == FAILURE)) { #else if (nxt_slow_path(php_request_startup(TSRMLS_C) == FAILURE)) { #endif - nxt_unit_req_debug(req, "php_request_startup() failed"); - rc = NXT_UNIT_ERROR; + nxt_unit_req_debug(ctx->req, "php_request_startup() failed"); - goto fail; + return NXT_ERROR; } - rc = NXT_UNIT_OK; - - php_execute_script(&file_handle TSRMLS_CC); - php_request_shutdown(NULL); - -fail: - - nxt_unit_request_done(req, rc); - - if (ctx->script_filename.start != nxt_php_script_filename.start) { - nxt_free(ctx->script_filename.start); - } + return NXT_OK; } -static int -nxt_php_startup(sapi_module_struct *sapi_module) +nxt_inline void +nxt_php_vcwd_chdir(nxt_unit_request_info_t *req, const nxt_str_t *dir) { - return php_module_startup(sapi_module, NULL, 0); + if (nxt_slow_path(VCWD_CHDIR((char *) dir->start) != 0)) { + nxt_unit_req_alert(req, "failed to VCWD_CHDIR(%V) %E", dir, nxt_errno); + } } @@ -1005,6 +1170,41 @@ nxt_php_set_str(nxt_unit_request_info_t *req, const char *name, } +#ifdef NXT_PHP7 + +static void * +nxt_php_hash_str_find_ptr(const HashTable *ht, const nxt_str_t *str) +{ + return zend_hash_str_find_ptr(ht, (const char *) str->start, str->length); +} + +#else + +static void * +nxt_php_hash_str_find_ptr(const HashTable *ht, const nxt_str_t *str) +{ + int ret; + void *entry; + char buf[256]; + + if (nxt_slow_path(str->length >= (sizeof(buf) - 1))) { + return NULL; + } + + nxt_memcpy(buf, str->start, str->length); + buf[str->length] = '\0'; + + ret = zend_hash_find(ht, buf, str->length + 1, &entry); + if (nxt_fast_path(ret == SUCCESS)) { + return entry; + } + + return NULL; +} + +#endif + + static void nxt_php_set_cstr(nxt_unit_request_info_t *req, const char *name, const char *cstr, uint32_t len, zval *track_vars_array TSRMLS_DC) -- cgit From a60f856ce2bc2eccbce0b0dfaa6ec98a30f74f67 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Tue, 3 Mar 2020 20:37:47 +0300 Subject: Improved validation of the "action" object. Now it enforces the mutual exclusivity of "pass", "proxy", and "share" options. --- src/nxt_conf_validation.c | 57 ++++++++++++++++++++++++++++++++++------------- test/test_routing.py | 2 -- 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 5a1f7839..4282f46b 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -323,17 +323,27 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_match_members[] = { }; -static nxt_conf_vldt_object_t nxt_conf_vldt_action_members[] = { +static nxt_conf_vldt_object_t nxt_conf_vldt_pass_action_members[] = { { nxt_string("pass"), NXT_CONF_VLDT_STRING, &nxt_conf_vldt_pass, NULL }, + NXT_CONF_VLDT_END +}; + + +static nxt_conf_vldt_object_t nxt_conf_vldt_share_action_members[] = { { nxt_string("share"), NXT_CONF_VLDT_STRING, NULL, NULL }, + NXT_CONF_VLDT_END +}; + + +static nxt_conf_vldt_object_t nxt_conf_vldt_proxy_action_members[] = { { nxt_string("proxy"), NXT_CONF_VLDT_STRING, &nxt_conf_vldt_proxy, @@ -912,30 +922,45 @@ static nxt_int_t nxt_conf_vldt_action(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) { - nxt_int_t ret; - nxt_conf_value_t *pass_value, *share_value, *proxy_value; + nxt_uint_t i; + nxt_conf_value_t *action; + nxt_conf_vldt_object_t *members; + + static struct { + nxt_str_t name; + nxt_conf_vldt_object_t *members; + + } actions[] = { + { nxt_string("pass"), nxt_conf_vldt_pass_action_members }, + { nxt_string("share"), nxt_conf_vldt_share_action_members }, + { nxt_string("proxy"), nxt_conf_vldt_proxy_action_members }, + }; - static nxt_str_t pass_str = nxt_string("pass"); - static nxt_str_t share_str = nxt_string("share"); - static nxt_str_t proxy_str = nxt_string("proxy"); + members = NULL; - ret = nxt_conf_vldt_object(vldt, value, nxt_conf_vldt_action_members); + for (i = 0; i < nxt_nitems(actions); i++) { + action = nxt_conf_get_object_member(value, &actions[i].name, NULL); - if (ret != NXT_OK) { - return ret; - } + if (action == NULL) { + continue; + } - pass_value = nxt_conf_get_object_member(value, &pass_str, NULL); - share_value = nxt_conf_get_object_member(value, &share_str, NULL); - proxy_value = nxt_conf_get_object_member(value, &proxy_str, NULL); + if (members != NULL) { + return nxt_conf_vldt_error(vldt, "The \"action\" object must have " + "just one of \"pass\", \"share\" or " + "\"proxy\" options set."); + } + + members = actions[i].members; + } - if (pass_value == NULL && share_value == NULL && proxy_value == NULL) { + if (members == NULL) { return nxt_conf_vldt_error(vldt, "The \"action\" object must have " - "either \"pass\" or \"share\" or " + "either \"pass\", \"share\", or " "\"proxy\" option set."); } - return NXT_OK; + return nxt_conf_vldt_object(vldt, value, members); } diff --git a/test/test_routing.py b/test/test_routing.py index 67f10d10..f03c0bb3 100644 --- a/test/test_routing.py +++ b/test/test_routing.py @@ -288,8 +288,6 @@ class TestRouting(TestApplicationProto): ) def test_routes_route_pass_absent(self): - self.skip_alerts.append(r'failed to apply new conf') - self.assertIn( 'error', self.conf([{"match": {"method": "GET"}, "action": {}}], 'routes'), -- cgit From a98de7f705f3b182fb405cced7ce85805bd6ffd3 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Tue, 3 Mar 2020 20:37:47 +0300 Subject: Added a "fallback" option to be used with the "share" action. It allows proceeding to another action if a file isn't available. An example: { "share": "/data/www/", "fallback": { "pass": "applications/php" } } In the example above, an attempt is made first to serve a request with a file from the "/data/www/" directory. If there's no such file, the request is passed to the "php" application. Fallback actions may be nested: { "share": "/data/www/", "fallback": { "share": "/data/cache/", "fallback": { "proxy": "http://127.0.0.1:9000" } } } --- src/nxt_conf_validation.c | 5 ++++ src/nxt_http.h | 1 + src/nxt_http_route.c | 72 ++++++++++++++++++++++++++++++++--------------- src/nxt_http_static.c | 13 +++++++++ 4 files changed, 68 insertions(+), 23 deletions(-) diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 4282f46b..8189cd44 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -339,6 +339,11 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_share_action_members[] = { NULL, NULL }, + { nxt_string("fallback"), + NXT_CONF_VLDT_OBJECT, + &nxt_conf_vldt_action, + NULL }, + NXT_CONF_VLDT_END }; diff --git a/src/nxt_http.h b/src/nxt_http.h index 030d77a7..67c7645b 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -189,6 +189,7 @@ struct nxt_http_action_s { nxt_http_route_t *route; nxt_http_upstream_t *upstream; nxt_app_t *application; + nxt_http_action_t *fallback; } u; nxt_str_t name; diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index ef9593b7..6fce895e 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -43,6 +43,7 @@ typedef struct { nxt_conf_value_t *pass; nxt_conf_value_t *share; nxt_conf_value_t *proxy; + nxt_conf_value_t *fallback; } nxt_http_route_action_conf_t; @@ -175,7 +176,7 @@ static nxt_http_route_t *nxt_http_route_create(nxt_task_t *task, static nxt_http_route_match_t *nxt_http_route_match_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_conf_value_t *cv); static nxt_int_t nxt_http_route_action_create(nxt_router_temp_conf_t *tmcf, - nxt_conf_value_t *cv, nxt_http_route_match_t *match); + nxt_conf_value_t *cv, nxt_http_action_t *action); static nxt_http_route_table_t *nxt_http_route_table_create(nxt_task_t *task, nxt_mp_t *mp, nxt_conf_value_t *table_cv, nxt_http_route_object_t object, nxt_bool_t case_sensitive); @@ -407,7 +408,7 @@ nxt_http_route_match_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, uint32_t n; nxt_mp_t *mp; nxt_int_t ret; - nxt_conf_value_t *match_conf; + nxt_conf_value_t *match_conf, *action_conf; nxt_http_route_test_t *test; nxt_http_route_rule_t *rule; nxt_http_route_table_t *table; @@ -416,6 +417,7 @@ nxt_http_route_match_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_http_route_match_conf_t mtcf; static nxt_str_t match_path = nxt_string("/match"); + static nxt_str_t action_path = nxt_string("/action"); match_conf = nxt_conf_get_path(cv, &match_path); @@ -433,7 +435,12 @@ nxt_http_route_match_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, match->action.handler = NULL; match->items = n; - ret = nxt_http_route_action_create(tmcf, cv, match); + action_conf = nxt_conf_get_path(cv, &action_path); + if (nxt_slow_path(action_conf == NULL)) { + return NULL; + } + + ret = nxt_http_route_action_create(tmcf, action_conf, &match->action); if (nxt_slow_path(ret != NXT_OK)) { return NULL; } @@ -579,30 +586,27 @@ static nxt_conf_map_t nxt_http_route_action_conf[] = { NXT_CONF_MAP_PTR, offsetof(nxt_http_route_action_conf_t, proxy) }, + { + nxt_string("fallback"), + NXT_CONF_MAP_PTR, + offsetof(nxt_http_route_action_conf_t, fallback) + }, }; static nxt_int_t nxt_http_route_action_create(nxt_router_temp_conf_t *tmcf, nxt_conf_value_t *cv, - nxt_http_route_match_t *match) + nxt_http_action_t *action) { nxt_mp_t *mp; nxt_int_t ret; nxt_str_t name, *string; - nxt_conf_value_t *conf, *action_conf; + nxt_conf_value_t *conf; nxt_http_route_action_conf_t accf; - static nxt_str_t action_path = nxt_string("/action"); - - action_conf = nxt_conf_get_path(cv, &action_path); - if (action_conf == NULL) { - return NXT_ERROR; - } - nxt_memzero(&accf, sizeof(accf)); - ret = nxt_conf_map_object(tmcf->mem_pool, - action_conf, nxt_http_route_action_conf, + ret = nxt_conf_map_object(tmcf->mem_pool, cv, nxt_http_route_action_conf, nxt_nitems(nxt_http_route_action_conf), &accf); if (ret != NXT_OK) { return ret; @@ -612,7 +616,7 @@ nxt_http_route_action_create(nxt_router_temp_conf_t *tmcf, nxt_conf_value_t *cv, if (accf.share != NULL) { conf = accf.share; - match->action.handler = nxt_http_static_handler; + action->handler = nxt_http_static_handler; } else if (accf.proxy != NULL) { conf = accf.proxy; @@ -622,13 +626,23 @@ nxt_http_route_action_create(nxt_router_temp_conf_t *tmcf, nxt_conf_value_t *cv, mp = tmcf->router_conf->mem_pool; - string = nxt_str_dup(mp, &match->action.name, &name); + string = nxt_str_dup(mp, &action->name, &name); if (nxt_slow_path(string == NULL)) { return NXT_ERROR; } + if (accf.fallback != NULL) { + action->u.fallback = nxt_mp_zalloc(mp, sizeof(nxt_http_action_t)); + if (nxt_slow_path(action->u.fallback == NULL)) { + return NXT_ERROR; + } + + return nxt_http_route_action_create(tmcf, accf.fallback, + action->u.fallback); + } + if (accf.proxy != NULL) { - return nxt_http_proxy_create(mp, &match->action); + return nxt_http_proxy_create(mp, action); } return NXT_OK; @@ -1043,18 +1057,13 @@ static void nxt_http_route_resolve(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_http_route_t *route) { - nxt_http_action_t *action; nxt_http_route_match_t **match, **end; match = &route->match[0]; end = match + route->items; while (match < end) { - action = &(*match)->action; - - if (action->handler == NULL) { - nxt_http_action_resolve(task, tmcf, &(*match)->action); - } + nxt_http_action_resolve(task, tmcf, &(*match)->action); match++; } @@ -1067,6 +1076,16 @@ nxt_http_action_resolve(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, { nxt_str_t name; + if (action->handler != NULL) { + if (action->handler == nxt_http_static_handler + && action->u.fallback != NULL) + { + nxt_http_action_resolve(task, tmcf, action->u.fallback); + } + + return; + } + name = action->name; if (nxt_str_start(&name, "applications/", 13)) { @@ -1201,6 +1220,13 @@ nxt_http_action_cleanup(nxt_task_t *task, nxt_http_action_t *action) { if (action->handler == nxt_http_application_handler) { nxt_router_app_use(task, action->u.application, -1); + return; + } + + if (action->handler == nxt_http_static_handler + && action->u.fallback != NULL) + { + nxt_http_action_cleanup(task, action->u.fallback); } } diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 44132859..46ae57a7 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -49,6 +49,10 @@ nxt_http_static_handler(nxt_task_t *task, nxt_http_request_t *r, if (nxt_slow_path(!nxt_str_eq(r->method, "GET", 3))) { if (!nxt_str_eq(r->method, "HEAD", 4)) { + if (action->u.fallback != NULL) { + return action->u.fallback; + } + nxt_http_request_error(task, r, NXT_HTTP_METHOD_NOT_ALLOWED); return NULL; } @@ -123,6 +127,10 @@ nxt_http_static_handler(nxt_task_t *task, nxt_http_request_t *r, break; } + if (level == NXT_LOG_ERR && action->u.fallback != NULL) { + return action->u.fallback; + } + if (status != NXT_HTTP_NOT_FOUND) { nxt_log(task, level, "open(\"%FN\") failed %E", f->name, f->error); } @@ -222,8 +230,13 @@ nxt_http_static_handler(nxt_task_t *task, nxt_http_request_t *r, nxt_file_close(task, f); if (nxt_slow_path(!nxt_is_dir(&fi))) { + if (action->u.fallback != NULL) { + return action->u.fallback; + } + nxt_log(task, NXT_LOG_ERR, "\"%FN\" is not a regular file", f->name); + nxt_http_request_error(task, r, NXT_HTTP_NOT_FOUND); return NULL; } -- cgit From 3b7b2fae54b7c11e979552f13f7f315775673981 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Tue, 3 Mar 2020 17:54:02 +0000 Subject: Tests: check unique options in "action" object. --- test/test_routing.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/test/test_routing.py b/test/test_routing.py index f03c0bb3..be00d733 100644 --- a/test/test_routing.py +++ b/test/test_routing.py @@ -294,6 +294,56 @@ class TestRouting(TestApplicationProto): 'route pass absent configure', ) + def test_routes_action_unique(self): + self.assertIn( + 'success', + self.conf( + { + "listeners": { + "*:7080": {"pass": "routes"}, + "*:7081": {"pass": "applications/app"}, + }, + "routes": [{"action": {"proxy": "http://127.0.0.1:7081"}}], + "applications": { + "app": { + "type": "python", + "processes": {"spare": 0}, + "path": "/app", + "module": "wsgi", + } + }, + } + ), + ) + + self.assertIn( + 'error', + self.conf( + {"proxy": "http://127.0.0.1:7081", "share": self.testdir}, + 'routes/0/action', + ), + 'proxy share', + ) + self.assertIn( + 'error', + self.conf( + { + "proxy": "http://127.0.0.1:7081", + "pass": "applications/app", + }, + 'routes/0/action', + ), + 'proxy pass', + ) + self.assertIn( + 'error', + self.conf( + {"share": self.testdir, "pass": "applications/app"}, + 'routes/0/action', + ), + 'share pass', + ) + def test_routes_rules_two(self): self.assertIn( 'success', -- cgit From 293b0da5201917dcecc09340e8e33eaff954b326 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Tue, 3 Mar 2020 18:17:31 +0000 Subject: Tests: added tests for "fallback" option for the "share" action. --- test/test_share_fallback.py | 212 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 test/test_share_fallback.py diff --git a/test/test_share_fallback.py b/test/test_share_fallback.py new file mode 100644 index 00000000..8c45793e --- /dev/null +++ b/test/test_share_fallback.py @@ -0,0 +1,212 @@ +import os +import unittest +from unit.applications.proto import TestApplicationProto + + +class TestStatic(TestApplicationProto): + prerequisites = {} + + def setUp(self): + super().setUp() + + os.makedirs(self.testdir + '/assets/dir') + with open(self.testdir + '/assets/index.html', 'w') as index: + index.write('0123456789') + + os.makedirs(self.testdir + '/assets/403') + os.chmod(self.testdir + '/assets/403', 0o000) + + self._load_conf( + { + "listeners": { + "*:7080": {"pass": "routes"}, + "*:7081": {"pass": "applications/empty"}, + }, + "routes": [{"action": {"share": self.testdir + "/assets"}}], + "applications": { + "empty": { + "type": "python", + "processes": {"spare": 0}, + "path": self.current_dir + "/python/empty", + "working_directory": self.current_dir + + "/python/empty", + "module": "wsgi", + } + }, + } + ) + + def tearDown(self): + os.chmod(self.testdir + '/assets/403', 0o777) + + super().tearDown() + + def test_fallback(self): + self.assertIn( + 'success', + self.conf({"share": "/blah"}, 'routes/0/action'), + 'configure bad path no fallback', + ) + self.assertEqual(self.get()['status'], 404, 'bad path no fallback') + + self.assertIn( + 'success', + self.conf( + {"share": "/blah", "fallback": {"pass": "applications/empty"}}, + 'routes/0/action', + ), + 'configure bad path fallback', + ) + resp = self.get() + self.assertEqual(resp['status'], 200, 'bad path fallback status') + self.assertEqual(resp['body'], '', 'bad path fallback') + + def test_fallback_valid_path(self): + self.assertIn( + 'success', + self.conf( + { + "share": self.testdir + "/assets", + "fallback": {"pass": "applications/empty"}, + }, + 'routes/0/action', + ), + 'configure fallback', + ) + resp = self.get() + self.assertEqual(resp['status'], 200, 'fallback status') + self.assertEqual(resp['body'], '0123456789', 'fallback') + + resp = self.get(url='/403/') + self.assertEqual(resp['status'], 200, 'fallback status 403') + self.assertEqual(resp['body'], '', 'fallback 403') + + resp = self.post() + self.assertEqual(resp['status'], 200, 'fallback status 405') + self.assertEqual(resp['body'], '', 'fallback 405') + + self.assertEqual( + self.get(url='/dir')['status'], 301, 'fallback status 301' + ) + + def test_fallback_nested(self): + self.assertIn( + 'success', + self.conf( + { + "share": "/blah", + "fallback": { + "share": "/blah/blah", + "fallback": {"pass": "applications/empty"}, + }, + }, + 'routes/0/action', + ), + 'configure fallback nested', + ) + resp = self.get() + self.assertEqual(resp['status'], 200, 'fallback nested status') + self.assertEqual(resp['body'], '', 'fallback nested') + + def test_fallback_share(self): + self.assertIn( + 'success', + self.conf( + { + "share": "/blah", + "fallback": {"share": self.testdir + "/assets"}, + }, + 'routes/0/action', + ), + 'configure fallback share', + ) + resp = self.get() + self.assertEqual(resp['status'], 200, 'fallback share status') + self.assertEqual(resp['body'], '0123456789', 'fallback share') + + resp = self.head() + self.assertEqual(resp['status'], 200, 'fallback share status HEAD') + self.assertEqual(resp['body'], '', 'fallback share HEAD') + + self.assertEqual( + self.get(url='/dir')['status'], 301, 'fallback share status 301' + ) + + def test_fallback_proxy(self): + self.assertIn( + 'success', + self.conf( + { + "share": "/blah", + "fallback": {"proxy": "http://127.0.0.1:7081"}, + }, + 'routes/0/action', + ), + 'configure fallback proxy', + ) + resp = self.get() + self.assertEqual(resp['status'], 200, 'fallback proxy status') + self.assertEqual(resp['body'], '', 'fallback proxy') + + @unittest.skip('not yet') + def test_fallback_proxy_cycle(self): + self.assertIn( + 'success', + self.conf( + { + "share": "/blah", + "fallback": {"proxy": "http://127.0.0.1:7080"}, + }, + 'routes/0/action', + ), + 'configure fallback cycle', + ) + self.assertNotEqual(self.get()['status'], 200, 'fallback cycle') + + self.assertIn( + 'success', self.conf_delete('listeners/*:7081'), 'delete listener' + ) + self.assertNotEqual(self.get()['status'], 200, 'fallback cycle 2') + + def test_fallback_invalid(self): + self.assertIn( + 'error', + self.conf({"share": "/blah", "fallback": {}}, 'routes/0/action'), + 'configure fallback empty', + ) + self.assertIn( + 'error', + self.conf({"share": "/blah", "fallback": ""}, 'routes/0/action'), + 'configure fallback not object', + ) + self.assertIn( + 'error', + self.conf( + { + "proxy": "http://127.0.0.1:7081", + "fallback": {"share": "/blah"}, + }, + 'routes/0/action', + ), + 'configure fallback proxy invalid', + ) + self.assertIn( + 'error', + self.conf( + { + "pass": "applications/empty", + "fallback": {"share": "/blah"}, + }, + 'routes/0/action', + ), + 'configure fallback pass invalid', + ) + self.assertIn( + 'error', + self.conf({"fallback": {"share": "/blah"}}, 'routes/0/action'), + 'configure fallback only', + ) + + +if __name__ == '__main__': + TestStatic.main() -- cgit From 80763b3e64cfeae358c521fa563cc9eaa48f4ea2 Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Tue, 3 Mar 2020 18:53:26 +0000 Subject: Tests: chdir() and open() for PHP module. These tests ensure optimizations in the chdir calls don't break SAPI semantics. --- test/php/cwd/index.php | 19 ++++++ test/php/cwd/subdir/index.php | 1 + test/php/open/index.php | 7 ++ test/php/open/test.txt | 1 + test/test_php_application.py | 135 +++++++++++++++++++++++++++++++++++-- test/unit/applications/lang/php.py | 4 +- 6 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 test/php/cwd/index.php create mode 100644 test/php/cwd/subdir/index.php create mode 100644 test/php/open/index.php create mode 100644 test/php/open/test.txt diff --git a/test/php/cwd/index.php b/test/php/cwd/index.php new file mode 100644 index 00000000..24ae3a21 --- /dev/null +++ b/test/php/cwd/index.php @@ -0,0 +1,19 @@ +opcache_enabled; +} + +header('X-OPcache: ' . $opcache); + +print(getcwd()); +?> diff --git a/test/php/cwd/subdir/index.php b/test/php/cwd/subdir/index.php new file mode 100644 index 00000000..597bcac4 --- /dev/null +++ b/test/php/cwd/subdir/index.php @@ -0,0 +1 @@ + diff --git a/test/php/open/index.php b/test/php/open/index.php new file mode 100644 index 00000000..a5bebc54 --- /dev/null +++ b/test/php/open/index.php @@ -0,0 +1,7 @@ + diff --git a/test/php/open/test.txt b/test/php/open/test.txt new file mode 100644 index 00000000..30d74d25 --- /dev/null +++ b/test/php/open/test.txt @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/test/test_php_application.py b/test/test_php_application.py index 37606fa2..c3645a99 100644 --- a/test/test_php_application.py +++ b/test/test_php_application.py @@ -13,6 +13,28 @@ class TestPHPApplication(TestApplicationPHP): self.assertRegex(body, r'time: \d+', 'disable_functions before time') self.assertRegex(body, r'exec: \/\w+', 'disable_functions before exec') + def set_opcache(self, app, val): + self.assertIn( + 'success', + self.conf( + { + "admin": { + "opcache.enable": val, + "opcache.enable_cli": val, + }, + }, + 'applications/' + app + '/options', + ), + ) + + opcache = self.get()['headers']['X-OPcache'] + + if not opcache or opcache == '-1': + print('opcache is not supported') + raise unittest.SkipTest() + + self.assertEqual(opcache, val, 'opcache value') + def test_php_application_variables(self): self.load('variables') @@ -466,7 +488,8 @@ class TestPHPApplication(TestApplicationPHP): def test_php_application_script(self): self.assertIn( - 'success', self.conf( + 'success', + self.conf( { "listeners": {"*:7080": {"pass": "applications/script"}}, "applications": { @@ -478,7 +501,8 @@ class TestPHPApplication(TestApplicationPHP): } }, } - ), 'configure script' + ), + 'configure script', ) resp = self.get() @@ -488,7 +512,8 @@ class TestPHPApplication(TestApplicationPHP): def test_php_application_index_default(self): self.assertIn( - 'success', self.conf( + 'success', + self.conf( { "listeners": {"*:7080": {"pass": "applications/phpinfo"}}, "applications": { @@ -499,7 +524,8 @@ class TestPHPApplication(TestApplicationPHP): } }, } - ), 'configure index default' + ), + 'configure index default', ) resp = self.get() @@ -541,6 +567,107 @@ class TestPHPApplication(TestApplicationPHP): str(resp['status']) + resp['body'], '200', 'status new root' ) + def run_php_application_cwd_root_tests(self): + self.assertIn( + 'success', self.conf_delete('applications/cwd/working_directory') + ) + + script_cwd = self.current_dir + '/php/cwd' + + resp = self.get() + self.assertEqual(resp['status'], 200, 'status ok') + self.assertEqual(resp['body'], script_cwd, 'default cwd') + + self.assertIn( + 'success', + self.conf( + '"' + self.current_dir + '"', + 'applications/cwd/working_directory', + ), + ) + + resp = self.get() + self.assertEqual(resp['status'], 200, 'status ok') + self.assertEqual(resp['body'], script_cwd, 'wdir cwd') + + resp = self.get(url='/?chdir=/') + self.assertEqual(resp['status'], 200, 'status ok') + self.assertEqual(resp['body'], '/', 'cwd after chdir') + + # cwd must be restored + + resp = self.get() + self.assertEqual(resp['status'], 200, 'status ok') + self.assertEqual(resp['body'], script_cwd, 'cwd restored') + + resp = self.get(url='/subdir/') + self.assertEqual( + resp['body'], script_cwd + '/subdir', 'cwd subdir', + ) + + def test_php_application_cwd_root(self): + self.load('cwd') + self.run_php_application_cwd_root_tests() + + def test_php_application_cwd_opcache_disabled(self): + self.load('cwd') + self.set_opcache('cwd', '0') + self.run_php_application_cwd_root_tests() + + def test_php_application_cwd_opcache_enabled(self): + self.load('cwd') + self.set_opcache('cwd', '1') + self.run_php_application_cwd_root_tests() + + def run_php_application_cwd_script_tests(self): + self.load('cwd') + + script_cwd = self.current_dir + '/php/cwd' + + self.assertIn( + 'success', self.conf_delete('applications/cwd/working_directory') + ) + + self.assertIn( + 'success', self.conf('"index.php"', 'applications/cwd/script') + ) + + self.assertEqual( + self.get()['body'], script_cwd, 'default cwd', + ) + + self.assertEqual( + self.get(url='/?chdir=/')['body'], '/', 'cwd after chdir', + ) + + # cwd must be restored + self.assertEqual(self.get()['body'], script_cwd, 'cwd restored') + + def test_php_application_cwd_script(self): + self.load('cwd') + self.run_php_application_cwd_script_tests() + + def test_php_application_cwd_script_opcache_disabled(self): + self.load('cwd') + self.set_opcache('cwd', '0') + self.run_php_application_cwd_script_tests() + + def test_php_application_cwd_script_opcache_enabled(self): + self.load('cwd') + self.set_opcache('cwd', '1') + self.run_php_application_cwd_script_tests() + + def test_php_application_path_relative(self): + self.load('open') + + self.assertEqual(self.get()['body'], 'test', 'relative path') + + self.assertNotEqual( + self.get(url='/?chdir=/')['body'], 'test', 'relative path w/ chdir' + ) + + self.assertEqual(self.get()['body'], 'test', 'relative path 2') + if __name__ == '__main__': TestPHPApplication.main() diff --git a/test/unit/applications/lang/php.py b/test/unit/applications/lang/php.py index 6b1677e6..e8c70c62 100644 --- a/test/unit/applications/lang/php.py +++ b/test/unit/applications/lang/php.py @@ -4,7 +4,7 @@ from unit.applications.proto import TestApplicationProto class TestApplicationPHP(TestApplicationProto): application_type = "php" - def load(self, script, name='index.php', **kwargs): + def load(self, script, index='index.php', **kwargs): script_path = self.current_dir + '/php/' + script self._load_conf( @@ -16,7 +16,7 @@ class TestApplicationPHP(TestApplicationProto): "processes": {"spare": 0}, "root": script_path, "working_directory": script_path, - "index": name, + "index": index, } }, }, -- cgit From 75cb2a947d7194c9ef69f9e1b9dedaae2cf1905f Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Wed, 4 Mar 2020 15:24:27 +0300 Subject: PHP: rearranged feature checks in ./configure. Now it prints version even if PHP was built without embed SAPI. --- auto/modules/php | 94 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/auto/modules/php b/auto/modules/php index 51b068e7..014bb3e7 100644 --- a/auto/modules/php +++ b/auto/modules/php @@ -110,76 +110,78 @@ if /bin/sh -c "${NXT_PHP_CONFIG} --version" >> $NXT_AUTOCONF_ERR 2>&1; then fi fi - nxt_feature="PHP embed SAPI" - nxt_feature_name="" - nxt_feature_run=no - nxt_feature_incs="${NXT_PHP_INCLUDE}" - nxt_feature_libs="${NXT_PHP_LIB} ${NXT_PHP_LDFLAGS}" - nxt_feature_test=" - #include - #include - - int main() { - php_module_startup(NULL, NULL, 0); - return 0; - }" - - . auto/feature - - if [ $nxt_found = no ]; then - $echo - $echo $0: error: no PHP embed SAPI found. - $echo - exit 1; - fi +else + $echo + $echo $0: error: no PHP found. + $echo + exit 1; +fi - # Bug #71041 (https://bugs.php.net/bug.php?id=71041). - nxt_feature="PHP zend_signal_startup()" - nxt_feature_name="" - nxt_feature_run=no - nxt_feature_incs="${NXT_PHP_INCLUDE}" - nxt_feature_libs="${NXT_PHP_LIB} ${NXT_PHP_LDFLAGS}" - nxt_feature_test=" - #include - #include +nxt_feature="PHP version" +nxt_feature_name="" +nxt_feature_run=value +nxt_feature_incs="${NXT_PHP_INCLUDE}" +nxt_feature_libs="${NXT_PHP_LIB} ${NXT_PHP_LDFLAGS}" +nxt_feature_test=" + #include - int main() { - zend_signal_startup(); - return 0; - }" + int main() { + printf(\"%s\", PHP_VERSION); + return 0; + }" - . auto/feature +. auto/feature - if [ $nxt_found = yes ]; then - NXT_ZEND_SIGNAL_STARTUP=1 - else - NXT_ZEND_SIGNAL_STARTUP=0 - fi -else +nxt_feature="PHP embed SAPI" +nxt_feature_name="" +nxt_feature_run=no +nxt_feature_incs="${NXT_PHP_INCLUDE}" +nxt_feature_libs="${NXT_PHP_LIB} ${NXT_PHP_LDFLAGS}" +nxt_feature_test=" + #include + #include + + int main() { + php_module_startup(NULL, NULL, 0); + return 0; + }" + +. auto/feature + +if [ $nxt_found = no ]; then $echo - $echo $0: error: no PHP found. + $echo $0: error: no PHP embed SAPI found. $echo exit 1; fi -nxt_feature="PHP version" +# Bug #71041 (https://bugs.php.net/bug.php?id=71041). + +nxt_feature="PHP zend_signal_startup()" nxt_feature_name="" -nxt_feature_run=value +nxt_feature_run=no nxt_feature_incs="${NXT_PHP_INCLUDE}" nxt_feature_libs="${NXT_PHP_LIB} ${NXT_PHP_LDFLAGS}" nxt_feature_test=" #include + #include int main() { - printf(\"%s\", PHP_VERSION); + zend_signal_startup(); return 0; }" . auto/feature +if [ $nxt_found = yes ]; then + NXT_ZEND_SIGNAL_STARTUP=1 +else + NXT_ZEND_SIGNAL_STARTUP=0 +fi + if grep ^$NXT_PHP_MODULE: $NXT_MAKEFILE 2>&1 > /dev/null; then $echo -- cgit From afa2f86ecf3ff99b598b14b3448c056b914cd32a Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Wed, 4 Mar 2020 15:24:27 +0300 Subject: PHP: added ZTS indication to ./configure output. --- auto/modules/php | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/auto/modules/php b/auto/modules/php index 014bb3e7..e2e5498a 100644 --- a/auto/modules/php +++ b/auto/modules/php @@ -158,6 +158,25 @@ if [ $nxt_found = no ]; then fi +nxt_feature="PHP Zend Thread Safety" +nxt_feature_name="" +nxt_feature_run=no +nxt_feature_incs="${NXT_PHP_INCLUDE}" +nxt_feature_libs="${NXT_PHP_LIB} ${NXT_PHP_LDFLAGS}" +nxt_feature_test=" + #include + #include + + int main() { + #ifndef ZTS + #error ZTS is not defined. + #endif + return 0; + }" + +. auto/feature + + # Bug #71041 (https://bugs.php.net/bug.php?id=71041). nxt_feature="PHP zend_signal_startup()" -- cgit From 2d0dca52431c08f5f7650ce7bad5a1bc680fa150 Mon Sep 17 00:00:00 2001 From: Igor Sysoev Date: Wed, 4 Mar 2020 14:03:30 +0300 Subject: The kqueue EOF flag might be ignored on some conditions. If kqueue reported both the EVFILT_READ and the EVFILT_WRITE events for the socket but only the former had the EV_EOF flag set, the flag was silently ignored. --- src/nxt_kqueue_engine.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nxt_kqueue_engine.c b/src/nxt_kqueue_engine.c index 9edbc346..ecc3251e 100644 --- a/src/nxt_kqueue_engine.c +++ b/src/nxt_kqueue_engine.c @@ -747,7 +747,7 @@ nxt_kqueue_poll(nxt_event_engine_t *engine, nxt_msec_t timeout) err = kev->fflags; eof = (kev->flags & EV_EOF) != 0; ev->kq_errno = err; - ev->kq_eof = eof; + ev->kq_eof |= eof; if (ev->read <= NXT_EVENT_BLOCKED) { nxt_debug(ev->task, "blocked read event fd:%d", ev->fd); @@ -778,7 +778,7 @@ nxt_kqueue_poll(nxt_event_engine_t *engine, nxt_msec_t timeout) err = kev->fflags; eof = (kev->flags & EV_EOF) != 0; ev->kq_errno = err; - ev->kq_eof = eof; + ev->kq_eof |= eof; if (ev->write <= NXT_EVENT_BLOCKED) { nxt_debug(ev->task, "blocked write event fd:%d", ev->fd); -- cgit From 36578c7b43c5872277ba880b28a8a8a237ab7841 Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Wed, 4 Mar 2020 13:42:08 +0000 Subject: PHP: fixed log format in alert. Found by Coverity: CID 354832 and CID 354833. --- src/nxt_php_sapi.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 3702c371..f5053652 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -892,7 +892,8 @@ nxt_inline void nxt_php_vcwd_chdir(nxt_unit_request_info_t *req, const nxt_str_t *dir) { if (nxt_slow_path(VCWD_CHDIR((char *) dir->start) != 0)) { - nxt_unit_req_alert(req, "failed to VCWD_CHDIR(%V) %E", dir, nxt_errno); + nxt_unit_req_alert(req, "VCWD_CHDIR(%s) failed (%d: %s)", + dir->start, errno, strerror(errno)); } } -- cgit From 3617d4ed03326239587b7299370e64669533c6f5 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Fri, 6 Mar 2020 20:08:29 +0300 Subject: Tests: simplified unitd process running. There are no reasons to wrap the Unit daemon in a separate Python process. --- test/unit/main.py | 68 +++++++++++++++++++++---------------------------------- 1 file changed, 26 insertions(+), 42 deletions(-) diff --git a/test/unit/main.py b/test/unit/main.py index 149cfb2c..322066ba 100644 --- a/test/unit/main.py +++ b/test/unit/main.py @@ -5,6 +5,7 @@ import stat import time import fcntl import shutil +import signal import argparse import platform import tempfile @@ -210,22 +211,19 @@ class TestUnit(unittest.TestCase): print() - self._p = Process(target=subprocess.call, args=[ [ - self.unitd, - '--no-daemon', - '--modules', self.pardir + '/build', - '--state', self.testdir + '/state', - '--pid', self.testdir + '/unit.pid', - '--log', self.testdir + '/unit.log', - '--control', 'unix:' + self.testdir + '/control.unit.sock', - ] ]) - self._p.start() - - if not self.waitforfiles( - self.testdir + '/unit.pid', - self.testdir + '/unit.log', - self.testdir + '/control.unit.sock', - ): + self._p = subprocess.Popen( + [ + self.unitd, + '--no-daemon', + '--modules', self.pardir + '/build', + '--state', self.testdir + '/state', + '--pid', self.testdir + '/unit.pid', + '--log', self.testdir + '/unit.log', + '--control', 'unix:' + self.testdir + '/control.unit.sock', + ] + ) + + if not self.waitforfiles(self.testdir + '/control.unit.sock'): exit("Could not start unit") self._started = True @@ -238,35 +236,21 @@ class TestUnit(unittest.TestCase): self.skip_sanitizer = False def _stop(self): - with open(self.testdir + '/unit.pid', 'r') as f: - pid = f.read().rstrip() - - subprocess.call(['kill', '-s', 'QUIT', pid]) - - for i in range(150): - if not os.path.exists(self.testdir + '/unit.pid'): - break - time.sleep(0.1) - - self._p.join(timeout=5) - - if self._p.is_alive(): - self._p.terminate() - self._p.join(timeout=5) - - if self._p.is_alive(): - self.fail("Could not terminate process " + str(self._p.pid)) - - if os.path.exists(self.testdir + '/unit.pid'): - self.fail("Could not terminate unit") + with self._p as p: + p.send_signal(signal.SIGQUIT) + + try: + retcode = p.wait(15) + if retcode: + self.fail( + "Child process terminated with code " + str(retcode) + ) + except: + self.fail("Could not terminate unit") + p.kill() self._started = False - if self._p.exitcode: - self.fail( - "Child process terminated with code " + str(self._p.exitcode) - ) - def _check_alerts(self, log): found = False -- cgit From 810b8dbb6798bd8ddcbafae7ecd9e5ee535c92f4 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Fri, 6 Mar 2020 20:08:38 +0300 Subject: Tests: unitd stderr output redirected to unit.log. A part of the debug log was printed to stderr before the log file was opened. Now, this output is redirected to the same log file. --- test/unit/main.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/unit/main.py b/test/unit/main.py index 322066ba..a7981c1c 100644 --- a/test/unit/main.py +++ b/test/unit/main.py @@ -209,19 +209,19 @@ class TestUnit(unittest.TestCase): os.mkdir(self.testdir + '/state') - print() - - self._p = subprocess.Popen( - [ - self.unitd, - '--no-daemon', - '--modules', self.pardir + '/build', - '--state', self.testdir + '/state', - '--pid', self.testdir + '/unit.pid', - '--log', self.testdir + '/unit.log', - '--control', 'unix:' + self.testdir + '/control.unit.sock', - ] - ) + with open(self.testdir + '/unit.log', 'w') as log: + self._p = subprocess.Popen( + [ + self.unitd, + '--no-daemon', + '--modules', self.pardir + '/build', + '--state', self.testdir + '/state', + '--pid', self.testdir + '/unit.pid', + '--log', self.testdir + '/unit.log', + '--control', 'unix:' + self.testdir + '/control.unit.sock', + ], + stderr=log, + ) if not self.waitforfiles(self.testdir + '/control.unit.sock'): exit("Could not start unit") -- cgit From f36f0f2461a7f0447e92f977ff9c0c71fcb08ffb Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Tue, 10 Mar 2020 18:10:42 +0000 Subject: Tests: redirect tests output to the stdout. --- test/run.py | 2 +- test/test_access_log.py | 1 - test/test_tls.py | 18 ++++++++++++------ test/unit/applications/lang/go.py | 5 +++-- test/unit/applications/lang/java.py | 4 ++-- test/unit/applications/tls.py | 3 ++- 6 files changed, 20 insertions(+), 13 deletions(-) diff --git a/test/run.py b/test/run.py index b79d0484..59e06bcb 100755 --- a/test/run.py +++ b/test/run.py @@ -12,7 +12,7 @@ if __name__ == '__main__': tests = loader.discover(start_dir=this_dir) suite.addTests(tests) - runner = unittest.TextTestRunner(verbosity=3) + runner = unittest.TextTestRunner(stream=sys.stdout, verbosity=3) result = runner.run(suite) ret = not (len(result.failures) == len(result.errors) == 0) diff --git a/test/test_access_log.py b/test/test_access_log.py index 94f6e7bf..898d8b24 100644 --- a/test/test_access_log.py +++ b/test/test_access_log.py @@ -2,7 +2,6 @@ import os import re import time import unittest -from subprocess import call from unit.applications.lang.python import TestApplicationPython diff --git a/test/test_tls.py b/test/test_tls.py index 1ead111c..475e9919 100644 --- a/test/test_tls.py +++ b/test/test_tls.py @@ -157,7 +157,8 @@ class TestTLS(TestApplicationTLS): '-genkey', '-out', self.testdir + '/ec.key', '-name', 'prime256v1', - ] + ], + stderr=subprocess.STDOUT, ) subprocess.call( @@ -170,7 +171,8 @@ class TestTLS(TestApplicationTLS): '-config', self.testdir + '/openssl.conf', '-key', self.testdir + '/ec.key', '-out', self.testdir + '/ec.crt', - ] + ], + stderr=subprocess.STDOUT, ) self.certificate_load('ec') @@ -230,7 +232,8 @@ class TestTLS(TestApplicationTLS): '-config', self.testdir + '/openssl.conf', '-out', self.testdir + '/int.csr', '-keyout', self.testdir + '/int.key', - ] + ], + stderr=subprocess.STDOUT, ) subprocess.call( @@ -242,7 +245,8 @@ class TestTLS(TestApplicationTLS): '-config', self.testdir + '/openssl.conf', '-out', self.testdir + '/end.csr', '-keyout', self.testdir + '/end.key', - ] + ], + stderr=subprocess.STDOUT, ) with open(self.testdir + '/ca.conf', 'w') as f: @@ -288,7 +292,8 @@ basicConstraints = critical,CA:TRUE""" '-cert', self.testdir + '/root.crt', '-in', self.testdir + '/int.csr', '-out', self.testdir + '/int.crt', - ] + ], + stderr=subprocess.STDOUT, ) subprocess.call( @@ -302,7 +307,8 @@ basicConstraints = critical,CA:TRUE""" '-cert', self.testdir + '/int.crt', '-in', self.testdir + '/end.csr', '-out', self.testdir + '/end.crt', - ] + ], + stderr=subprocess.STDOUT, ) crt_path = self.testdir + '/end-int.crt' diff --git a/test/unit/applications/lang/go.py b/test/unit/applications/lang/go.py index 7212a95c..e0f83c0a 100644 --- a/test/unit/applications/lang/go.py +++ b/test/unit/applications/lang/go.py @@ -1,5 +1,5 @@ import os -from subprocess import Popen +import subprocess from unit.applications.proto import TestApplicationProto @@ -26,7 +26,7 @@ class TestApplicationGo(TestApplicationProto): env['GOPATH'] = self.pardir + '/build/go' try: - process = Popen( + process = subprocess.Popen( [ 'go', 'build', @@ -35,6 +35,7 @@ class TestApplicationGo(TestApplicationProto): self.current_dir + '/go/' + script + '/' + name + '.go', ], env=env, + stderr=subprocess.STDOUT, ) process.communicate() diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py index a370d96b..a8a09ce5 100644 --- a/test/unit/applications/lang/java.py +++ b/test/unit/applications/lang/java.py @@ -1,7 +1,7 @@ import os import glob import shutil -from subprocess import Popen +import subprocess from unit.applications.proto import TestApplicationProto @@ -64,7 +64,7 @@ class TestApplicationJava(TestApplicationProto): javac.extend(src) try: - process = Popen(javac) + process = subprocess.Popen(javac, stderr=subprocess.STDOUT) process.communicate() except: diff --git a/test/unit/applications/tls.py b/test/unit/applications/tls.py index 1290279d..9213974a 100644 --- a/test/unit/applications/tls.py +++ b/test/unit/applications/tls.py @@ -47,7 +47,8 @@ class TestApplicationTLS(TestApplicationProto): '-config', self.testdir + '/openssl.conf', '-out', self.testdir + '/' + name + '.crt', '-keyout', self.testdir + '/' + name + '.key', - ] + ], + stderr=subprocess.STDOUT, ) if load: -- cgit From f092b093f561f0bffeb990c176246c6f6413401d Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Tue, 10 Mar 2020 18:13:47 +0000 Subject: Tests: use blocking to print unit.log files. --- test/unit/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/main.py b/test/unit/main.py index a7981c1c..b38d87e2 100644 --- a/test/unit/main.py +++ b/test/unit/main.py @@ -371,7 +371,7 @@ class TestUnit(unittest.TestCase): # set stdout to non-blocking - if TestUnit.detailed: + if TestUnit.detailed or TestUnit.print_log: fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, 0) def _print_log(self, data=None): -- cgit From f302ed067017f233e41b9ca823a72d8b0fa1aa93 Mon Sep 17 00:00:00 2001 From: Axel Duch Date: Wed, 11 Mar 2020 14:18:39 +0000 Subject: Fixed negative patterns combined with address rules. --- src/nxt_http_route.c | 46 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index 6fce895e..05eaa6d4 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -192,6 +192,7 @@ static nxt_http_route_rule_t *nxt_http_route_rule_create(nxt_task_t *task, nxt_mp_t *mp, nxt_conf_value_t *cv, nxt_bool_t case_sensitive, nxt_http_route_pattern_case_t pattern_case); static int nxt_http_pattern_compare(const void *one, const void *two); +static int nxt_http_addr_pattern_compare(const void *one, const void *two); static nxt_int_t nxt_http_route_pattern_create(nxt_task_t *task, nxt_mp_t *mp, nxt_conf_value_t *cv, nxt_http_route_pattern_t *pattern, nxt_http_route_pattern_case_t pattern_case); @@ -881,6 +882,12 @@ nxt_http_route_addr_rule_create(nxt_task_t *task, nxt_mp_t *mp, } } + if (n > 1) { + nxt_qsort(addr_rule->addr_pattern, addr_rule->items, + sizeof(nxt_http_route_addr_pattern_t), + nxt_http_addr_pattern_compare); + } + return addr_rule; } @@ -904,6 +911,18 @@ nxt_http_pattern_compare(const void *one, const void *two) } +static int +nxt_http_addr_pattern_compare(const void *one, const void *two) +{ + const nxt_http_route_addr_pattern_t *p1, *p2; + + p1 = one; + p2 = two; + + return (p2->base.negative - p1->base.negative); +} + + static nxt_int_t nxt_http_route_pattern_create(nxt_task_t *task, nxt_mp_t *mp, nxt_conf_value_t *cv, nxt_http_route_pattern_t *pattern, @@ -1527,19 +1546,34 @@ static nxt_int_t nxt_http_route_addr_rule(nxt_http_request_t *r, nxt_http_route_addr_rule_t *addr_rule, nxt_sockaddr_t *sa) { - uint32_t i, n; + uint32_t n; + nxt_bool_t matches; nxt_http_route_addr_pattern_t *p; n = addr_rule->items; + p = &addr_rule->addr_pattern[0] - 1; - for (i = 0; i < n; i++) { - p = &addr_rule->addr_pattern[i]; - if (nxt_http_route_addr_pattern_match(p, sa)) { + do { + p++; + n--; + + matches = nxt_http_route_addr_pattern_match(p, sa); + + if (p->base.negative) { + if (matches) { + continue; + } + + return 0; + } + + if (matches) { return 1; } - } - return 0; + } while (n > 0); + + return p->base.negative; } -- cgit From b214b7c690f9383f296f3d77dcf26755ec00bac3 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Fri, 31 Jan 2020 18:12:16 +0000 Subject: Tests: more routing tests with negative rules. --- test/test_routing.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/test/test_routing.py b/test/test_routing.py index be00d733..950923d6 100644 --- a/test/test_routing.py +++ b/test/test_routing.py @@ -1412,6 +1412,13 @@ class TestRouting(TestApplicationProto): sock, port = sock_port() sock2, port2 = sock_port() + self.route_match({"source": ["*:" + str(port), "!127.0.0.1"]}) + self.assertEqual(self.get(sock=sock)['status'], 404, 'negative 3') + self.assertEqual(self.get(sock=sock2)['status'], 404, 'negative 4') + + sock, port = sock_port() + sock2, port2 = sock_port() + self.route_match( {"source": "127.0.0.1:" + str(port) + "-" + str(port)} ) @@ -1726,6 +1733,7 @@ class TestRouting(TestApplicationProto): self.route_match_invalid({"source": "127"}) self.route_match_invalid({"source": "256.0.0.1"}) self.route_match_invalid({"source": "127.0.0."}) + self.route_match_invalid({"source": " 127.0.0.1"}) self.route_match_invalid({"source": "127.0.0.1:"}) self.route_match_invalid({"source": "127.0.0.1/"}) self.route_match_invalid({"source": "11.0.0.0/33"}) @@ -1765,6 +1773,55 @@ class TestRouting(TestApplicationProto): self.assertEqual(self.get()['status'], 404, 'dest neg') self.assertEqual(self.get(port=7081)['status'], 200, 'dest neg 2') + self.route_match({"destination": ['!*:7080', '!*:7081']}) + self.assertEqual(self.get()['status'], 404, 'dest neg 3') + self.assertEqual(self.get(port=7081)['status'], 404, 'dest neg 4') + + self.route_match({"destination": ['!*:7081', '!*:7082']}) + self.assertEqual(self.get()['status'], 200, 'dest neg 5') + + self.route_match({"destination": ['*:7080', '!*:7080']}) + self.assertEqual(self.get()['status'], 404, 'dest neg 6') + + self.route_match( + {"destination": ['127.0.0.1:7080', '*:7081', '!*:7080']} + ) + self.assertEqual(self.get()['status'], 404, 'dest neg 7') + self.assertEqual(self.get(port=7081)['status'], 200, 'dest neg 8') + + self.route_match({"destination": ['!*:7081', '!*:7082', '*:7083']}) + self.assertEqual(self.get()['status'], 404, 'dest neg 9') + + self.route_match( + {"destination": ['*:7081', '!127.0.0.1:7080', '*:7080']} + ) + self.assertEqual(self.get()['status'], 404, 'dest neg 10') + self.assertEqual(self.get(port=7081)['status'], 200, 'dest neg 11') + + self.assertIn( + 'success', + self.conf_delete('routes/0/match/destination/0'), + 'remove destination rule', + ) + self.assertEqual(self.get()['status'], 404, 'dest neg 12') + self.assertEqual(self.get(port=7081)['status'], 404, 'dest neg 13') + + self.assertIn( + 'success', + self.conf_delete('routes/0/match/destination/0'), + 'remove destination rule 2', + ) + self.assertEqual(self.get()['status'], 200, 'dest neg 14') + self.assertEqual(self.get(port=7081)['status'], 404, 'dest neg 15') + + self.assertIn( + 'success', + self.conf_post("\"!127.0.0.1\"", 'routes/0/match/destination'), + 'add destination rule', + ) + self.assertEqual(self.get()['status'], 404, 'dest neg 16') + self.assertEqual(self.get(port=7081)['status'], 404, 'dest neg 17') + def test_routes_match_destination_proxy(self): self.assertIn( 'success', -- cgit From 643d4383fa7db466e1d80c7d8c350da4ec043f2b Mon Sep 17 00:00:00 2001 From: Igor Sysoev Date: Wed, 4 Mar 2020 14:03:32 +0300 Subject: Refactored nxt_http_action. --- src/nxt_http_route.c | 28 ++++++++++++---------------- src/nxt_router.c | 8 +++++--- src/nxt_router.h | 4 ++-- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index 05eaa6d4..df52894a 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -203,8 +203,8 @@ static void nxt_http_route_resolve(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_http_route_t *route); static void nxt_http_action_resolve(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_http_action_t *action); -static nxt_http_route_t *nxt_http_route_find(nxt_http_routes_t *routes, - nxt_str_t *name); +static void nxt_http_route_find(nxt_http_routes_t *routes, nxt_str_t *name, + nxt_http_action_t *action); static void nxt_http_route_cleanup(nxt_task_t *task, nxt_http_route_t *routes); static nxt_http_action_t *nxt_http_route_handler(nxt_task_t *task, @@ -1111,11 +1111,9 @@ nxt_http_action_resolve(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, name.length -= 13; name.start += 13; - action->u.application = nxt_router_listener_application(tmcf, &name); + nxt_router_listener_application(tmcf, &name, action); nxt_router_app_use(task, action->u.application, 1); - action->handler = nxt_http_application_handler; - } else if (nxt_str_start(&name, "routes", 6)) { if (name.length == 6) { @@ -1127,15 +1125,14 @@ nxt_http_action_resolve(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, name.start += 7; } - action->u.route = nxt_http_route_find(tmcf->router_conf->routes, &name); - - action->handler = nxt_http_route_handler; + nxt_http_route_find(tmcf->router_conf->routes, &name, action); } } -static nxt_http_route_t * -nxt_http_route_find(nxt_http_routes_t *routes, nxt_str_t *name) +static void +nxt_http_route_find(nxt_http_routes_t *routes, nxt_str_t *name, + nxt_http_action_t *action) { nxt_http_route_t **route, **end; @@ -1144,13 +1141,14 @@ nxt_http_route_find(nxt_http_routes_t *routes, nxt_str_t *name) while (route < end) { if (nxt_strstr_eq(&(*route)->name, name)) { - return *route; + action->u.route = *route; + action->handler = nxt_http_route_handler; + + return; } route++; } - - return NULL; } @@ -1191,11 +1189,9 @@ nxt_http_pass_application(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, action->name = *name; - action->u.application = nxt_router_listener_application(tmcf, name); + nxt_router_listener_application(tmcf, name, action); nxt_router_app_use(task, action->u.application, 1); - action->handler = nxt_http_application_handler; - return action; } diff --git a/src/nxt_router.c b/src/nxt_router.c index 6ef4ee78..46a6b921 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1904,8 +1904,9 @@ nxt_router_app_find(nxt_queue_t *queue, nxt_str_t *name) } -nxt_app_t * -nxt_router_listener_application(nxt_router_temp_conf_t *tmcf, nxt_str_t *name) +void +nxt_router_listener_application(nxt_router_temp_conf_t *tmcf, nxt_str_t *name, + nxt_http_action_t *action) { nxt_app_t *app; @@ -1915,7 +1916,8 @@ nxt_router_listener_application(nxt_router_temp_conf_t *tmcf, nxt_str_t *name) app = nxt_router_app_find(&tmcf->previous, name); } - return app; + action->u.application = app; + action->handler = nxt_http_application_handler; } diff --git a/src/nxt_router.h b/src/nxt_router.h index 1517c14b..9ce9f3be 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -218,8 +218,8 @@ void nxt_router_access_log_reopen_handler(nxt_task_t *task, void nxt_router_process_http_request(nxt_task_t *task, nxt_http_request_t *r, nxt_app_t *app); void nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port); -nxt_app_t *nxt_router_listener_application(nxt_router_temp_conf_t *tmcf, - nxt_str_t *name); +void nxt_router_listener_application(nxt_router_temp_conf_t *tmcf, + nxt_str_t *name, nxt_http_action_t *action); void nxt_router_app_use(nxt_task_t *task, nxt_app_t *app, int i); void nxt_router_listen_event_release(nxt_task_t *task, nxt_listen_event_t *lev, nxt_socket_conf_joint_t *joint); -- cgit From 794248090a74f31cbfcf24ea8c835df2d4d21073 Mon Sep 17 00:00:00 2001 From: Igor Sysoev Date: Wed, 4 Mar 2020 14:04:08 +0300 Subject: Legacy upstream code removed. --- src/nxt_main.h | 8 -- src/nxt_upstream.c | 36 -------- src/nxt_upstream.h | 36 -------- src/nxt_upstream_round_robin.c | 194 ----------------------------------------- 4 files changed, 274 deletions(-) diff --git a/src/nxt_main.h b/src/nxt_main.h index d9e337d2..b310c4fa 100644 --- a/src/nxt_main.h +++ b/src/nxt_main.h @@ -149,15 +149,7 @@ typedef void (*nxt_event_conn_handler_t)(nxt_thread_t *thr, nxt_conn_t *c); #include -#include -typedef struct nxt_upstream_source_s nxt_upstream_source_t; - #include -#include -#include -#include -#include -#include #include #include diff --git a/src/nxt_upstream.c b/src/nxt_upstream.c index e1615120..3899a3ce 100644 --- a/src/nxt_upstream.c +++ b/src/nxt_upstream.c @@ -5,39 +5,3 @@ */ #include - - -typedef struct { - void (*peer_get)(nxt_upstream_peer_t *up); - void (*peer_free)(nxt_upstream_peer_t *up); -} nxt_upstream_name_t; - - -static const nxt_upstream_name_t nxt_upstream_names[] = { - - { "round_robin", &nxt_upstream_round_robin }, -}; - - -void -nxt_upstream_create(nxt_upstream_peer_t *up) -{ - /* TODO: dynamic balancer add & lvlhsh */ - nxt_upstream_names[0].create(up); -} - - -void -nxt_upstream_peer(nxt_upstream_peer_t *up) -{ - nxt_upstream_t *u; - - u = up->upstream; - - if (u != NULL) { - u->peer_get(up); - return; - } - - nxt_upstream_create(up); -} diff --git a/src/nxt_upstream.h b/src/nxt_upstream.h index d1fca2a5..a25e0940 100644 --- a/src/nxt_upstream.h +++ b/src/nxt_upstream.h @@ -8,40 +8,4 @@ #define _NXT_UPSTREAM_H_INCLUDED_ -typedef struct nxt_upstream_peer_s nxt_upstream_peer_t; - - -struct nxt_upstream_peer_s { - /* STUB */ - void *upstream; - void *data; - /**/ - - nxt_sockaddr_t *sockaddr; - nxt_nsec_t delay; - - uint32_t tries; - in_port_t port; - - nxt_str_t addr; - nxt_mp_t *mem_pool; - void (*ready_handler)(nxt_task_t *task, nxt_upstream_peer_t *up); - - void (*protocol_handler)(nxt_upstream_source_t *us); -}; - - -typedef struct { - void (*ready_handler)(void *data); - nxt_work_handler_t completion_handler; - nxt_work_handler_t error_handler; -} nxt_upstream_state_t; - - -/* STUB */ -NXT_EXPORT void nxt_upstream_round_robin_peer(nxt_task_t *task, - nxt_upstream_peer_t *up); -/**/ - - #endif /* _NXT_UPSTREAM_H_INCLUDED_ */ diff --git a/src/nxt_upstream_round_robin.c b/src/nxt_upstream_round_robin.c index 09a3bce3..4d53c716 100644 --- a/src/nxt_upstream_round_robin.c +++ b/src/nxt_upstream_round_robin.c @@ -4,197 +4,3 @@ * Copyright (C) NGINX, Inc. */ -#include - - -typedef struct { - int32_t weight; - int32_t effective_weight; - int32_t current_weight; - uint32_t down; /* 1 bit */ - nxt_msec_t last_accessed; - nxt_sockaddr_t *sockaddr; -} nxt_upstream_round_robin_peer_t; - - -typedef struct { - nxt_uint_t npeers; - nxt_upstream_round_robin_peer_t *peers; - nxt_thread_spinlock_t lock; -} nxt_upstream_round_robin_t; - - -static void nxt_upstream_round_robin_create(nxt_task_t *task, void *obj, - void *data); -static void nxt_upstream_round_robin_peer_error(nxt_task_t *task, void *obj, - void *data); -static void nxt_upstream_round_robin_get_peer(nxt_task_t *task, - nxt_upstream_peer_t *up); - - -void -nxt_upstream_round_robin_peer(nxt_task_t *task, nxt_upstream_peer_t *up) -{ - nxt_job_sockaddr_parse_t *jbs; - - if (up->upstream != NULL) { - nxt_upstream_round_robin_get_peer(task, up); - } - - jbs = nxt_job_create(up->mem_pool, sizeof(nxt_job_sockaddr_parse_t)); - if (nxt_slow_path(jbs == NULL)) { - up->ready_handler(task, up); - return; - } - - jbs->resolve.job.task = task; - jbs->resolve.job.data = up; - jbs->resolve.port = up->port; - jbs->resolve.log_level = NXT_LOG_ERR; - jbs->resolve.ready_handler = nxt_upstream_round_robin_create; - jbs->resolve.error_handler = nxt_upstream_round_robin_peer_error; - jbs->addr = up->addr; - - nxt_job_sockaddr_parse(jbs); -} - - -static void -nxt_upstream_round_robin_create(nxt_task_t *task, void *obj, void *data) -{ - nxt_uint_t i; - nxt_sockaddr_t *sa; - nxt_upstream_peer_t *up; - nxt_job_sockaddr_parse_t *jbs; - nxt_upstream_round_robin_t *urr; - nxt_upstream_round_robin_peer_t *peer; - - jbs = obj; - up = jbs->resolve.job.data; - - urr = nxt_mp_zget(up->mem_pool, sizeof(nxt_upstream_round_robin_t)); - if (nxt_slow_path(urr == NULL)) { - goto fail; - } - - urr->npeers = jbs->resolve.count; - - peer = nxt_mp_zget(up->mem_pool, - urr->npeers * sizeof(nxt_upstream_round_robin_peer_t)); - if (nxt_slow_path(peer == NULL)) { - goto fail; - } - - urr->peers = peer; - - for (i = 0; i < urr->npeers; i++) { - peer[i].weight = 1; - peer[i].effective_weight = 1; - - sa = jbs->resolve.sockaddrs[i]; - - /* STUB */ - sa->type = SOCK_STREAM; - - nxt_sockaddr_text(sa); - - nxt_debug(task, "upstream peer: %*s", - (size_t) sa->length, nxt_sockaddr_start(sa)); - - /* TODO: memcpy to shared memory pool. */ - peer[i].sockaddr = sa; - } - - up->upstream = urr; - - /* STUB */ - up->sockaddr = peer[0].sockaddr; - - nxt_job_destroy(task, jbs); - up->ready_handler(task, up); - - //nxt_upstream_round_robin_get_peer(up); - return; - -fail: - - nxt_job_destroy(task, jbs); - - up->ready_handler(task, up); -} - - -static void -nxt_upstream_round_robin_peer_error(nxt_task_t *task, void *obj, void *data) -{ - nxt_upstream_peer_t *up; - nxt_job_sockaddr_parse_t *jbs; - - jbs = obj; - up = jbs->resolve.job.data; - - up->ready_handler(task, up); -} - - -static void -nxt_upstream_round_robin_get_peer(nxt_task_t *task, nxt_upstream_peer_t *up) -{ - int32_t effective_weights; - nxt_uint_t i; - nxt_msec_t now; - nxt_upstream_round_robin_t *urr; - nxt_upstream_round_robin_peer_t *peer, *best; - - urr = up->upstream; - - now = task->thread->engine->timers.now; - - nxt_thread_spin_lock(&urr->lock); - - best = NULL; - effective_weights = 0; - peer = urr->peers; - - for (i = 0; i < urr->npeers; i++) { - - if (peer[i].down) { - continue; - } - -#if 0 - if (peer[i].max_fails != 0 && peer[i].fails >= peer->max_fails) { - good = peer[i].last_accessed + peer[i].fail_timeout; - - if (nxt_msec_diff(now, peer[i].last_accessed) <= 0) { - continue; - } - } -#endif - - peer[i].current_weight += peer[i].effective_weight; - effective_weights += peer[i].effective_weight; - - if (peer[i].effective_weight < peer[i].weight) { - peer[i].effective_weight++; - } - - if (best == NULL || peer[i].current_weight > best->current_weight) { - best = &peer[i]; - } - } - - if (best != NULL) { - best->current_weight -= effective_weights; - best->last_accessed = now; - - up->sockaddr = best->sockaddr; - - } else { - up->sockaddr = NULL; - } - - nxt_thread_spin_unlock(&urr->lock); - - up->ready_handler(task, up); -} -- cgit From 7935ea45436ea832344cec945d39a61ae91f2a69 Mon Sep 17 00:00:00 2001 From: Igor Sysoev Date: Fri, 6 Mar 2020 18:28:54 +0300 Subject: Round robin upstream added. --- auto/sources | 1 + src/nxt_conf_validation.c | 130 +++++++++++++++++++++++++++++ src/nxt_h1proto.c | 3 +- src/nxt_http.h | 18 +++- src/nxt_http_proxy.c | 124 ++++++++++++++++++++------- src/nxt_http_route.c | 6 ++ src/nxt_router.c | 11 +++ src/nxt_router.h | 5 ++ src/nxt_upstream.c | 135 +++++++++++++++++++++++++++++- src/nxt_upstream.h | 70 ++++++++++++++++ src/nxt_upstream_round_robin.c | 185 +++++++++++++++++++++++++++++++++++++++++ 11 files changed, 654 insertions(+), 34 deletions(-) diff --git a/auto/sources b/auto/sources index 98e4a1f4..2283e543 100644 --- a/auto/sources +++ b/auto/sources @@ -69,6 +69,7 @@ NXT_LIB_SRCS=" \ src/nxt_job_resolve.c \ src/nxt_sockaddr.c \ src/nxt_listen_socket.c \ + src/nxt_upstream.c \ src/nxt_upstream_round_robin.c \ src/nxt_http_parse.c \ src/nxt_app_log.c \ diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 8189cd44..86c1dbcb 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -110,6 +110,12 @@ static nxt_int_t nxt_conf_vldt_java_classpath(nxt_conf_validation_t *vldt, nxt_conf_value_t *value); static nxt_int_t nxt_conf_vldt_java_option(nxt_conf_validation_t *vldt, nxt_conf_value_t *value); +static nxt_int_t nxt_conf_vldt_upstream(nxt_conf_validation_t *vldt, + nxt_str_t *name, nxt_conf_value_t *value); +static nxt_int_t nxt_conf_vldt_server(nxt_conf_validation_t *vldt, + nxt_str_t *name, nxt_conf_value_t *value); +static nxt_int_t nxt_conf_vldt_server_weight(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_isolation(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); @@ -226,6 +232,11 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_root_members[] = { &nxt_conf_vldt_object_iterator, (void *) &nxt_conf_vldt_app }, + { nxt_string("upstreams"), + NXT_CONF_VLDT_OBJECT, + &nxt_conf_vldt_object_iterator, + (void *) &nxt_conf_vldt_upstream }, + { nxt_string("access_log"), NXT_CONF_VLDT_STRING, NULL, @@ -682,6 +693,26 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_java_members[] = { }; +static nxt_conf_vldt_object_t nxt_conf_vldt_upstream_members[] = { + { nxt_string("servers"), + NXT_CONF_VLDT_OBJECT, + &nxt_conf_vldt_object_iterator, + (void *) &nxt_conf_vldt_server }, + + NXT_CONF_VLDT_END +}; + + +static nxt_conf_vldt_object_t nxt_conf_vldt_upstream_server_members[] = { + { nxt_string("weight"), + NXT_CONF_VLDT_INTEGER, + &nxt_conf_vldt_server_weight, + NULL }, + + NXT_CONF_VLDT_END +}; + + nxt_int_t nxt_conf_validate(nxt_conf_validation_t *vldt) { @@ -1017,6 +1048,27 @@ nxt_conf_vldt_pass(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, return NXT_OK; } + if (nxt_str_eq(&first, "upstreams", 9)) { + + if (second.length == 0) { + goto error; + } + + value = nxt_conf_get_object_member(vldt->conf, &first, NULL); + + if (nxt_slow_path(value == NULL)) { + goto error; + } + + value = nxt_conf_get_object_member(value, &second, NULL); + + if (nxt_slow_path(value == NULL)) { + goto error; + } + + return NXT_OK; + } + if (nxt_str_eq(&first, "routes", 6)) { value = nxt_conf_get_object_member(vldt->conf, &first, NULL); @@ -1901,3 +1953,81 @@ nxt_conf_vldt_java_option(nxt_conf_validation_t *vldt, nxt_conf_value_t *value) return NXT_OK; } + + +static nxt_int_t +nxt_conf_vldt_upstream(nxt_conf_validation_t *vldt, nxt_str_t *name, + nxt_conf_value_t *value) +{ + nxt_int_t ret; + nxt_conf_value_t *conf; + + static nxt_str_t servers = nxt_string("servers"); + + ret = nxt_conf_vldt_type(vldt, name, value, NXT_CONF_VLDT_OBJECT); + + if (ret != NXT_OK) { + return ret; + } + + ret = nxt_conf_vldt_object(vldt, value, nxt_conf_vldt_upstream_members); + + if (ret != NXT_OK) { + return ret; + } + + conf = nxt_conf_get_object_member(value, &servers, NULL); + if (conf == NULL) { + return nxt_conf_vldt_error(vldt, "The \"%V\" upstream must contain " + "\"servers\" object value.", name); + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_conf_vldt_server(nxt_conf_validation_t *vldt, nxt_str_t *name, + nxt_conf_value_t *value) +{ + nxt_int_t ret; + nxt_sockaddr_t *sa; + + ret = nxt_conf_vldt_type(vldt, name, value, NXT_CONF_VLDT_OBJECT); + + if (ret != NXT_OK) { + return ret; + } + + sa = nxt_sockaddr_parse(vldt->pool, name); + + if (sa == NULL) { + return nxt_conf_vldt_error(vldt, "The \"%V\" is not valid " + "server address.", name); + } + + return nxt_conf_vldt_object(vldt, value, + nxt_conf_vldt_upstream_server_members); +} + + +static nxt_int_t +nxt_conf_vldt_server_weight(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data) +{ + int64_t int_value; + + int_value = nxt_conf_get_integer(value); + + if (int_value <= 0) { + return nxt_conf_vldt_error(vldt, "The \"weight\" number must be " + "greater than 0."); + } + + if (int_value > NXT_INT32_T_MAX) { + return nxt_conf_vldt_error(vldt, "The \"weight\" number must " + "not exceed %d.", NXT_INT32_T_MAX); + } + + return NXT_OK; +} diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 8ce57893..1a6c537e 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -2004,7 +2005,7 @@ nxt_h1p_peer_connect(nxt_task_t *task, nxt_http_peer_t *peer) c->read_timer.task = task; c->write_timer.task = task; c->socket.data = peer; - c->remote = peer->sockaddr; + c->remote = peer->server->sockaddr; c->socket.write_ready = 1; c->write_state = &nxt_h1p_peer_connect_state; diff --git a/src/nxt_http.h b/src/nxt_http.h index 67c7645b..0e0694e5 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -106,10 +106,12 @@ typedef struct { } nxt_http_response_t; +typedef struct nxt_upstream_server_s nxt_upstream_server_t; + typedef struct { nxt_http_proto_t proto; nxt_http_request_t *request; - nxt_sockaddr_t *sockaddr; + nxt_upstream_server_t *server; nxt_list_t *fields; nxt_buf_t *body; nxt_off_t remainder; @@ -178,7 +180,6 @@ struct nxt_http_request_s { typedef struct nxt_http_route_s nxt_http_route_t; -typedef struct nxt_http_upstream_s nxt_http_upstream_t; struct nxt_http_action_s { @@ -187,9 +188,10 @@ struct nxt_http_action_s { nxt_http_action_t *action); union { nxt_http_route_t *route; - nxt_http_upstream_t *upstream; nxt_app_t *application; nxt_http_action_t *fallback; + nxt_upstream_t *upstream; + uint32_t upstream_number; } u; nxt_str_t name; @@ -275,6 +277,11 @@ nxt_http_action_t *nxt_http_pass_application(nxt_task_t *task, void nxt_http_routes_cleanup(nxt_task_t *task, nxt_http_routes_t *routes); void nxt_http_action_cleanup(nxt_task_t *task, nxt_http_action_t *action); +nxt_int_t nxt_upstreams_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, + nxt_conf_value_t *conf); +nxt_int_t nxt_upstreams_joint_create(nxt_router_temp_conf_t *tmcf, + nxt_upstream_t ***upstream_joint); + nxt_http_action_t *nxt_http_static_handler(nxt_task_t *task, nxt_http_request_t *r, nxt_http_action_t *action); nxt_int_t nxt_http_static_mtypes_init(nxt_mp_t *mp, nxt_lvlhsh_t *hash); @@ -285,6 +292,11 @@ nxt_str_t *nxt_http_static_mtypes_hash_find(nxt_lvlhsh_t *hash, nxt_http_action_t *nxt_http_application_handler(nxt_task_t *task, nxt_http_request_t *r, nxt_http_action_t *action); +void nxt_upstream_find(nxt_upstreams_t *upstreams, nxt_str_t *name, + nxt_http_action_t *action); +nxt_http_action_t *nxt_upstream_proxy_handler(nxt_task_t *task, + nxt_http_request_t *r, nxt_upstream_t *upstream); + nxt_int_t nxt_http_proxy_create(nxt_mp_t *mp, nxt_http_action_t *action); nxt_int_t nxt_http_proxy_date(void *ctx, nxt_http_field_t *field, diff --git a/src/nxt_http_proxy.c b/src/nxt_http_proxy.c index 7f4eeff2..893e9303 100644 --- a/src/nxt_http_proxy.c +++ b/src/nxt_http_proxy.c @@ -6,23 +6,21 @@ #include #include +#include -typedef void (*nxt_http_upstream_connect_t)(nxt_task_t *task, - nxt_http_upstream_t *upstream, nxt_http_peer_t *peer); - - -struct nxt_http_upstream_s { - uint32_t current; - uint32_t n; - uint8_t protocol; - nxt_http_upstream_connect_t connect; - nxt_sockaddr_t *sockaddr[1]; +struct nxt_upstream_proxy_s { + nxt_sockaddr_t *sockaddr; + uint8_t protocol; }; -static void nxt_http_upstream_connect(nxt_task_t *task, - nxt_http_upstream_t *upstream, nxt_http_peer_t *peer); +static void nxt_http_proxy_server_get(nxt_task_t *task, + nxt_upstream_server_t *us); +static void nxt_http_proxy_upstream_ready(nxt_task_t *task, + nxt_upstream_server_t *us); +static void nxt_http_proxy_upstream_error(nxt_task_t *task, + nxt_upstream_server_t *us); static nxt_http_action_t *nxt_http_proxy_handler(nxt_task_t *task, nxt_http_request_t *r, nxt_http_action_t *action); static void nxt_http_proxy_header_send(nxt_task_t *task, void *obj, void *data); @@ -43,12 +41,24 @@ static const nxt_http_request_state_t nxt_http_proxy_header_read_state; static const nxt_http_request_state_t nxt_http_proxy_read_state; +static const nxt_upstream_server_proto_t nxt_upstream_simple_proto = { + .get = nxt_http_proxy_server_get, +}; + + +static const nxt_upstream_peer_state_t nxt_upstream_proxy_state = { + .ready = nxt_http_proxy_upstream_ready, + .error = nxt_http_proxy_upstream_error, +}; + + nxt_int_t nxt_http_proxy_create(nxt_mp_t *mp, nxt_http_action_t *action) { - nxt_str_t name; - nxt_sockaddr_t *sa; - nxt_http_upstream_t *upstream; + nxt_str_t name; + nxt_sockaddr_t *sa; + nxt_upstream_t *up; + nxt_upstream_proxy_t *proxy; sa = NULL; name = action->name; @@ -66,18 +76,25 @@ nxt_http_proxy_create(nxt_mp_t *mp, nxt_http_action_t *action) } if (sa != NULL) { - upstream = nxt_mp_alloc(mp, sizeof(nxt_http_upstream_t)); - if (nxt_slow_path(upstream == NULL)) { + up = nxt_mp_alloc(mp, sizeof(nxt_upstream_t)); + if (nxt_slow_path(up == NULL)) { + return NXT_ERROR; + } + + up->name.length = sa->length; + up->name.start = nxt_sockaddr_start(sa); + up->proto = &nxt_upstream_simple_proto; + + proxy = nxt_mp_alloc(mp, sizeof(nxt_upstream_proxy_t)); + if (nxt_slow_path(proxy == NULL)) { return NXT_ERROR; } - upstream->current = 0; - upstream->n = 1; - upstream->protocol = NXT_HTTP_PROTO_H1; - upstream->connect = nxt_http_upstream_connect; - upstream->sockaddr[0] = sa; + proxy->sockaddr = sa; + proxy->protocol = NXT_HTTP_PROTO_H1; + up->type.proxy = proxy; - action->u.upstream = upstream; + action->u.upstream = up; action->handler = nxt_http_proxy_handler; } @@ -89,7 +106,22 @@ static nxt_http_action_t * nxt_http_proxy_handler(nxt_task_t *task, nxt_http_request_t *r, nxt_http_action_t *action) { - nxt_http_peer_t *peer; + return nxt_upstream_proxy_handler(task, r, action->u.upstream); +} + + +nxt_http_action_t * +nxt_upstream_proxy_handler(nxt_task_t *task, nxt_http_request_t *r, + nxt_upstream_t *upstream) +{ + nxt_http_peer_t *peer; + nxt_upstream_server_t *us; + + us = nxt_mp_zalloc(r->mem_pool, sizeof(nxt_upstream_server_t)); + if (nxt_slow_path(us == NULL)) { + nxt_http_request_error(task, r, NXT_HTTP_INTERNAL_SERVER_ERROR); + return NULL; + } peer = nxt_mp_zalloc(r->mem_pool, sizeof(nxt_http_peer_t)); if (nxt_slow_path(peer == NULL)) { @@ -102,18 +134,39 @@ nxt_http_proxy_handler(nxt_task_t *task, nxt_http_request_t *r, nxt_mp_retain(r->mem_pool); - action->u.upstream->connect(task, action->u.upstream, peer); + us->state = &nxt_upstream_proxy_state; + us->peer.http = peer; + peer->server = us; + + us->upstream = upstream; + upstream->proto->get(task, us); return NULL; } static void -nxt_http_upstream_connect(nxt_task_t *task, nxt_http_upstream_t *upstream, - nxt_http_peer_t *peer) +nxt_http_proxy_server_get(nxt_task_t *task, nxt_upstream_server_t *us) { - peer->protocol = upstream->protocol; - peer->sockaddr = upstream->sockaddr[0]; + nxt_upstream_proxy_t *proxy; + + proxy = us->upstream->type.proxy; + + us->sockaddr = proxy->sockaddr; + us->protocol = proxy->protocol; + + us->state->ready(task, us); +} + + +static void +nxt_http_proxy_upstream_ready(nxt_task_t *task, nxt_upstream_server_t *us) +{ + nxt_http_peer_t *peer; + + peer = us->peer.http; + + peer->protocol = us->protocol; peer->request->state = &nxt_http_proxy_header_send_state; @@ -121,6 +174,19 @@ nxt_http_upstream_connect(nxt_task_t *task, nxt_http_upstream_t *upstream, } +static void +nxt_http_proxy_upstream_error(nxt_task_t *task, nxt_upstream_server_t *us) +{ + nxt_http_request_t *r; + + r = us->peer.http->request; + + nxt_mp_release(r->mem_pool); + + nxt_http_request_error(task, r, NXT_HTTP_BAD_GATEWAY); +} + + static const nxt_http_request_state_t nxt_http_proxy_header_send_state nxt_aligned(64) = { diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index df52894a..d7f20bcb 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -1114,6 +1114,12 @@ nxt_http_action_resolve(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_router_listener_application(tmcf, &name, action); nxt_router_app_use(task, action->u.application, 1); + } else if (nxt_str_start(&name, "upstreams/", 10)) { + name.length -= 10; + name.start += 10; + + nxt_upstream_find(tmcf->router_conf->upstreams, &name, action); + } else if (nxt_str_start(&name, "routes", 6)) { if (name.length == 6) { diff --git a/src/nxt_router.c b/src/nxt_router.c index 46a6b921..d77ffa2b 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1634,6 +1634,11 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, tmcf->router_conf->routes = routes; } + ret = nxt_upstreams_create(task, tmcf, conf); + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + http = nxt_conf_get_path(conf, &http_path); #if 0 if (http == NULL) { @@ -2526,6 +2531,7 @@ nxt_router_engine_joints_create(nxt_router_temp_conf_t *tmcf, nxt_router_engine_conf_t *recf, nxt_queue_t *sockets, nxt_work_handler_t handler) { + nxt_int_t ret; nxt_joint_job_t *job; nxt_queue_link_t *qlk; nxt_socket_conf_t *skcf; @@ -2559,6 +2565,11 @@ nxt_router_engine_joints_create(nxt_router_temp_conf_t *tmcf, job->work.data = joint; + ret = nxt_upstreams_joint_create(tmcf, &joint->upstreams); + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + joint->count = 1; skcf = nxt_queue_link_data(qlk, nxt_socket_conf_t, link); diff --git a/src/nxt_router.h b/src/nxt_router.h index 9ce9f3be..85ef9a6c 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -18,6 +18,8 @@ typedef struct nxt_http_request_s nxt_http_request_t; typedef struct nxt_http_action_s nxt_http_action_t; typedef struct nxt_http_routes_s nxt_http_routes_t; +typedef struct nxt_upstream_s nxt_upstream_t; +typedef struct nxt_upstreams_s nxt_upstreams_t; typedef struct nxt_router_access_log_s nxt_router_access_log_t; @@ -43,6 +45,7 @@ typedef struct { nxt_router_t *router; nxt_http_routes_t *routes; + nxt_upstreams_t *upstreams; nxt_lvlhsh_t mtypes_hash; @@ -196,6 +199,8 @@ typedef struct { nxt_event_engine_t *engine; nxt_socket_conf_t *socket_conf; + nxt_upstream_t **upstreams; + /* Modules configuraitons. */ } nxt_socket_conf_joint_t; diff --git a/src/nxt_upstream.c b/src/nxt_upstream.c index 3899a3ce..66b6619a 100644 --- a/src/nxt_upstream.c +++ b/src/nxt_upstream.c @@ -4,4 +4,137 @@ * Copyright (C) NGINX, Inc. */ -#include +#include +#include +#include + + +static nxt_http_action_t *nxt_upstream_handler(nxt_task_t *task, + nxt_http_request_t *r, nxt_http_action_t *action); + + +nxt_int_t +nxt_upstreams_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, + nxt_conf_value_t *conf) +{ + size_t size; + uint32_t i, n, next; + nxt_mp_t *mp; + nxt_int_t ret; + nxt_str_t name, *string; + nxt_upstreams_t *upstreams; + nxt_conf_value_t *upstreams_conf, *upcf; + + static nxt_str_t upstreams_name = nxt_string("upstreams"); + + upstreams_conf = nxt_conf_get_object_member(conf, &upstreams_name, NULL); + + if (upstreams_conf == NULL) { + return NXT_OK; + } + + n = nxt_conf_object_members_count(upstreams_conf); + + if (n == 0) { + return NXT_OK; + } + + mp = tmcf->router_conf->mem_pool; + size = sizeof(nxt_upstreams_t) + n * sizeof(nxt_upstream_t); + + upstreams = nxt_mp_zalloc(mp, size); + if (nxt_slow_path(upstreams == NULL)) { + return NXT_ERROR; + } + + upstreams->items = n; + next = 0; + + for (i = 0; i < n; i++) { + upcf = nxt_conf_next_object_member(upstreams_conf, &name, &next); + + string = nxt_str_dup(mp, &upstreams->upstream[i].name, &name); + if (nxt_slow_path(string == NULL)) { + return NXT_ERROR; + } + + ret = nxt_upstream_round_robin_create(task, tmcf, upcf, + &upstreams->upstream[i]); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + } + + tmcf->router_conf->upstreams = upstreams; + + return NXT_OK; +} + + +void +nxt_upstream_find(nxt_upstreams_t *upstreams, nxt_str_t *name, + nxt_http_action_t *action) +{ + uint32_t i, n; + nxt_upstream_t *upstream; + + upstream = &upstreams->upstream[0]; + n = upstreams->items; + + for (i = 0; i < n; i++) { + if (nxt_strstr_eq(&upstream[i].name, name)) { + action->u.upstream_number = i; + action->handler = nxt_upstream_handler; + + return; + } + } +} + + +nxt_int_t +nxt_upstreams_joint_create(nxt_router_temp_conf_t *tmcf, + nxt_upstream_t ***upstream_joint) +{ + uint32_t i, n; + nxt_upstream_t *u, **up; + nxt_upstreams_t *upstreams; + nxt_router_conf_t *router_conf; + + router_conf = tmcf->router_conf; + upstreams = router_conf->upstreams; + + if (upstreams == NULL) { + *upstream_joint = NULL; + return NXT_OK; + } + + n = upstreams->items; + + up = nxt_mp_zalloc(router_conf->mem_pool, n * sizeof(nxt_upstream_t *)); + if (nxt_slow_path(up == NULL)) { + return NXT_ERROR; + } + + u = &upstreams->upstream[0]; + + for (i = 0; i < n; i++) { + up[i] = u[i].proto->joint_create(tmcf, &u[i]); + if (nxt_slow_path(up[i] == NULL)) { + return NXT_ERROR; + } + } + + *upstream_joint = up; + + return NXT_OK; +} + + +static nxt_http_action_t * +nxt_upstream_handler(nxt_task_t *task, nxt_http_request_t *r, + nxt_http_action_t *action) +{ + return nxt_upstream_proxy_handler(task, r, + r->conf->upstreams[action->u.upstream_number]); +} diff --git a/src/nxt_upstream.h b/src/nxt_upstream.h index a25e0940..afc53774 100644 --- a/src/nxt_upstream.h +++ b/src/nxt_upstream.h @@ -8,4 +8,74 @@ #define _NXT_UPSTREAM_H_INCLUDED_ +typedef struct nxt_upstream_proxy_s nxt_upstream_proxy_t; +typedef struct nxt_upstream_round_robin_s nxt_upstream_round_robin_t; +typedef struct nxt_upstream_round_robin_server_s + nxt_upstream_round_robin_server_t; + + +typedef void (*nxt_upstream_peer_ready_t)(nxt_task_t *task, + nxt_upstream_server_t *us); +typedef void (*nxt_upstream_peer_error_t)(nxt_task_t *task, + nxt_upstream_server_t *us); + + +typedef struct { + nxt_upstream_peer_ready_t ready; + nxt_upstream_peer_error_t error; +} nxt_upstream_peer_state_t; + + +typedef nxt_upstream_t *(*nxt_upstream_joint_create_t)( + nxt_router_temp_conf_t *tmcf, nxt_upstream_t *upstream); +typedef void (*nxt_upstream_server_get_t)(nxt_task_t *task, + nxt_upstream_server_t *us); + + +typedef struct { + nxt_upstream_joint_create_t joint_create; + nxt_upstream_server_get_t get; +} nxt_upstream_server_proto_t; + + +struct nxt_upstream_s { + const nxt_upstream_server_proto_t *proto; + + union { + nxt_upstream_proxy_t *proxy; + nxt_upstream_round_robin_t *round_robin; + } type; + + nxt_str_t name; +}; + + +struct nxt_upstreams_s { + uint32_t items; + nxt_upstream_t upstream[0]; +}; + + +struct nxt_upstream_server_s { + nxt_sockaddr_t *sockaddr; + const nxt_upstream_peer_state_t *state; + nxt_upstream_t *upstream; + + uint8_t protocol; + + union { + nxt_upstream_round_robin_server_t *round_robin; + } server; + + union { + nxt_http_peer_t *http; + } peer; +}; + + +nxt_int_t nxt_upstream_round_robin_create(nxt_task_t *task, + nxt_router_temp_conf_t *tmcf, nxt_conf_value_t *upstream_conf, + nxt_upstream_t *upstream); + + #endif /* _NXT_UPSTREAM_H_INCLUDED_ */ diff --git a/src/nxt_upstream_round_robin.c b/src/nxt_upstream_round_robin.c index 4d53c716..fd76ecb5 100644 --- a/src/nxt_upstream_round_robin.c +++ b/src/nxt_upstream_round_robin.c @@ -4,3 +4,188 @@ * Copyright (C) NGINX, Inc. */ +#include +#include +#include + + +struct nxt_upstream_round_robin_server_s { + nxt_sockaddr_t *sockaddr; + + int32_t current_weight; + int32_t effective_weight; + int32_t weight; + + uint8_t protocol; +}; + + +struct nxt_upstream_round_robin_s { + uint32_t items; + nxt_upstream_round_robin_server_t server[0]; +}; + + +static nxt_upstream_t *nxt_upstream_round_robin_joint_create( + nxt_router_temp_conf_t *tmcf, nxt_upstream_t *upstream); +static void nxt_upstream_round_robin_server_get(nxt_task_t *task, + nxt_upstream_server_t *us); + + +static const nxt_upstream_server_proto_t nxt_upstream_round_robin_proto = { + .joint_create = nxt_upstream_round_robin_joint_create, + .get = nxt_upstream_round_robin_server_get, +}; + + +static nxt_conf_map_t nxt_upstream_round_robin_server_conf[] = { + { + nxt_string("weight"), + NXT_CONF_MAP_INT32, + offsetof(nxt_upstream_round_robin_server_t, weight), + }, +}; + + +nxt_int_t +nxt_upstream_round_robin_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, + nxt_conf_value_t *upstream_conf, nxt_upstream_t *upstream) +{ + size_t size; + uint32_t i, n, next; + nxt_mp_t *mp; + nxt_str_t name; + nxt_sockaddr_t *sa; + nxt_conf_value_t *servers_conf, *srvcf; + nxt_upstream_round_robin_t *urr; + + static nxt_str_t servers = nxt_string("servers"); + + mp = tmcf->router_conf->mem_pool; + + servers_conf = nxt_conf_get_object_member(upstream_conf, &servers, NULL); + n = nxt_conf_object_members_count(servers_conf); + + size = sizeof(nxt_upstream_round_robin_t) + + n * sizeof(nxt_upstream_round_robin_server_t); + + urr = nxt_mp_zalloc(mp, size); + if (nxt_slow_path(urr == NULL)) { + return NXT_ERROR; + } + + urr->items = n; + next = 0; + + for (i = 0; i < n; i++) { + srvcf = nxt_conf_next_object_member(servers_conf, &name, &next); + + sa = nxt_sockaddr_parse(mp, &name); + if (nxt_slow_path(sa == NULL)) { + return NXT_ERROR; + } + + sa->type = SOCK_STREAM; + + urr->server[i].sockaddr = sa; + urr->server[i].weight = 1; + urr->server[i].protocol = NXT_HTTP_PROTO_H1; + + nxt_conf_map_object(mp, srvcf, nxt_upstream_round_robin_server_conf, + nxt_nitems(nxt_upstream_round_robin_server_conf), + &urr->server[i]); + + urr->server[i].effective_weight = urr->server[i].weight; + } + + upstream->proto = &nxt_upstream_round_robin_proto; + upstream->type.round_robin = urr; + + return NXT_OK; +} + + +static nxt_upstream_t * +nxt_upstream_round_robin_joint_create(nxt_router_temp_conf_t *tmcf, + nxt_upstream_t *upstream) +{ + size_t size; + uint32_t i, n; + nxt_mp_t *mp; + nxt_upstream_t *u; + nxt_upstream_round_robin_t *urr, *urrcf; + + mp = tmcf->router_conf->mem_pool; + + u = nxt_mp_alloc(mp, sizeof(nxt_upstream_t)); + if (nxt_slow_path(u == NULL)) { + return NULL; + } + + *u = *upstream; + + urrcf = upstream->type.round_robin; + + size = sizeof(nxt_upstream_round_robin_t) + + urrcf->items * sizeof(nxt_upstream_round_robin_server_t); + + urr = nxt_mp_alloc(mp, size); + if (nxt_slow_path(urr == NULL)) { + return NULL; + } + + u->type.round_robin = urr; + + n = urrcf->items; + urr->items = n; + + for (i = 0; i < n; i++) { + urr->server[i] = urrcf->server[i]; + } + + return u; +} + + +static void +nxt_upstream_round_robin_server_get(nxt_task_t *task, nxt_upstream_server_t *us) +{ + int32_t total; + uint32_t i, n; + nxt_upstream_round_robin_t *round_robin; + nxt_upstream_round_robin_server_t *s, *best; + + best = NULL; + total = 0; + + round_robin = us->upstream->type.round_robin; + + s = round_robin->server; + n = round_robin->items; + + for (i = 0; i < n; i++) { + + s[i].current_weight += s[i].effective_weight; + total += s[i].effective_weight; + + if (s[i].effective_weight < s[i].weight) { + s[i].effective_weight++; + } + + if (best == NULL || s[i].current_weight > best->current_weight) { + best = &s[i]; + } + } + + if (best == NULL) { + us->state->error(task, us); + return; + } + + best->current_weight -= total; + us->sockaddr = best->sockaddr; + us->protocol = best->protocol; + us->server.round_robin = best; + + us->state->ready(task, us); +} -- cgit From 961674af4220ee2e769bb01ce9acd213ecaaa33c Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Thu, 12 Mar 2020 13:44:30 +0000 Subject: Tests: skip "last message send failed" alerts globally. --- test/unit/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/main.py b/test/unit/main.py index b38d87e2..a7f5816a 100644 --- a/test/unit/main.py +++ b/test/unit/main.py @@ -230,6 +230,7 @@ class TestUnit(unittest.TestCase): self.skip_alerts = [ r'read signalfd\(4\) failed', + r'last message send failed', r'sendmsg.+failed', r'recvmsg.+failed', ] -- cgit From 1d752777674cacde1badc644102f393871aecfdc Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Thu, 12 Mar 2020 13:47:43 +0000 Subject: Tests: round robin upstream tests. --- test/python/upstreams/0/wsgi.py | 8 + test/python/upstreams/1/wsgi.py | 8 + test/python/upstreams/2/wsgi.py | 8 + test/test_upstreams_rr.py | 465 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 489 insertions(+) create mode 100644 test/python/upstreams/0/wsgi.py create mode 100644 test/python/upstreams/1/wsgi.py create mode 100644 test/python/upstreams/2/wsgi.py create mode 100644 test/test_upstreams_rr.py diff --git a/test/python/upstreams/0/wsgi.py b/test/python/upstreams/0/wsgi.py new file mode 100644 index 00000000..2c88979b --- /dev/null +++ b/test/python/upstreams/0/wsgi.py @@ -0,0 +1,8 @@ +import time + +def application(env, start_response): + delay = int(env.get('HTTP_X_DELAY', 0)) + + start_response('200', [('Content-Length', '0'), ('X-Upstream', '0')]) + time.sleep(delay) + return [] diff --git a/test/python/upstreams/1/wsgi.py b/test/python/upstreams/1/wsgi.py new file mode 100644 index 00000000..5077bdb1 --- /dev/null +++ b/test/python/upstreams/1/wsgi.py @@ -0,0 +1,8 @@ +import time + +def application(env, start_response): + delay = int(env.get('HTTP_X_DELAY', 0)) + + start_response('200', [('Content-Length', '0'), ('X-Upstream', '1')]) + time.sleep(delay) + return [] diff --git a/test/python/upstreams/2/wsgi.py b/test/python/upstreams/2/wsgi.py new file mode 100644 index 00000000..bb0ce797 --- /dev/null +++ b/test/python/upstreams/2/wsgi.py @@ -0,0 +1,8 @@ +import time + +def application(env, start_response): + delay = int(env.get('HTTP_X_DELAY', 0)) + + start_response('200', [('Content-Length', '0'), ('X-Upstream', '2')]) + time.sleep(delay) + return [] diff --git a/test/test_upstreams_rr.py b/test/test_upstreams_rr.py new file mode 100644 index 00000000..2bc2d90a --- /dev/null +++ b/test/test_upstreams_rr.py @@ -0,0 +1,465 @@ +import os +import re +import unittest +from unit.applications.lang.python import TestApplicationPython + + +class TestUpstreamsRR(TestApplicationPython): + prerequisites = {'modules': ['python']} + + def setUp(self): + super().setUp() + + self.assertIn( + 'success', + self.conf( + { + "listeners": { + "*:7080": {"pass": "upstreams/one"}, + "*:7081": {"pass": "applications/ups_0"}, + "*:7082": {"pass": "applications/ups_1"}, + "*:7083": {"pass": "applications/ups_2"}, + "*:7090": {"pass": "upstreams/two"}, + }, + "upstreams": { + "one": { + "servers": { + "127.0.0.1:7081": {}, + "127.0.0.1:7082": {}, + }, + }, + "two": { + "servers": { + "127.0.0.1:7081": {}, + "127.0.0.1:7082": {}, + }, + }, + }, + "applications": { + "ups_0": { + "type": "python", + "processes": {"spare": 0}, + "path": self.current_dir + "/python/upstreams/0", + "working_directory": self.current_dir + + "/python/upstreams/0", + "module": "wsgi", + }, + "ups_1": { + "type": "python", + "processes": {"spare": 0}, + "path": self.current_dir + "/python/upstreams/1", + "working_directory": self.current_dir + + "/python/upstreams/1", + "module": "wsgi", + }, + "ups_2": { + "type": "python", + "processes": {"spare": 0}, + "path": self.current_dir + "/python/upstreams/2", + "working_directory": self.current_dir + + "/python/upstreams/2", + "module": "wsgi", + }, + }, + }, + ), + 'upstreams initial configuration', + ) + + self.cpu_count = os.cpu_count() + + def get_resps(self, req=100, port=7080): + resps = [0] + for _ in range(req): + headers = self.get(port=port)['headers'] + if 'X-Upstream' in headers: + ups = int(headers['X-Upstream']) + + if ups > len(resps) - 1: + resps.extend([0] * (ups - len(resps) + 1)) + + resps[ups] += 1 + + return resps + + def get_resps_sc(self, req=100, port=7080): + to_send = b"""GET / HTTP/1.1 +Host: localhost + +""" * ( + req - 1 + ) + + to_send += b"""GET / HTTP/1.1 +Host: localhost +Connection: close + +""" + + resp = self.http(to_send, raw_resp=True, raw=True, port=port) + ups = re.findall('X-Upstream: (\d+)', resp) + resps = [0] * (int(max(ups)) + 1) + + for i in range(len(ups)): + resps[int(ups[i])] += 1 + + return resps + + def test_upstreams_rr_no_weight(self): + resps = self.get_resps() + self.assertLessEqual( + abs(resps[0] - resps[1]), self.cpu_count, 'no weight' + ) + + self.assertIn( + 'success', + self.conf_delete('upstreams/one/servers/127.0.0.1:7081'), + 'no weight server remove', + ) + + resps = self.get_resps(req=50) + self.assertEqual(resps[1], 50, 'no weight 2') + + self.assertIn( + 'success', + self.conf({}, 'upstreams/one/servers/127.0.0.1:7081'), + 'no weight server revert', + ) + + resps = self.get_resps() + self.assertLessEqual( + abs(resps[0] - resps[1]), self.cpu_count, 'no weight 3' + ) + + self.assertIn( + 'success', + self.conf({}, 'upstreams/one/servers/127.0.0.1:7083'), + 'no weight server new', + ) + + resps = self.get_resps() + self.assertLessEqual( + max(resps) - min(resps), self.cpu_count, 'no weight 4' + ) + + resps = self.get_resps_sc(req=30) + self.assertEqual(resps[0], 10, 'no weight 4 0') + self.assertEqual(resps[1], 10, 'no weight 4 1') + self.assertEqual(resps[2], 10, 'no weight 4 2') + + def test_upstreams_rr_weight(self): + self.assertIn( + 'success', + self.conf({"weight": 3}, 'upstreams/one/servers/127.0.0.1:7081'), + 'configure weight', + ) + + resps = self.get_resps_sc() + self.assertEqual(resps[0], 75, 'weight 3 0') + self.assertEqual(resps[1], 25, 'weight 3 1') + + self.assertIn( + 'success', + self.conf_delete('upstreams/one/servers/127.0.0.1:7081/weight'), + 'configure weight remove', + ) + resps = self.get_resps_sc(req=10) + self.assertEqual(resps[0], 5, 'weight 0 0') + self.assertEqual(resps[1], 5, 'weight 0 1') + + self.assertIn( + 'success', + self.conf('1', 'upstreams/one/servers/127.0.0.1:7081/weight'), + 'configure weight 1', + ) + + resps = self.get_resps_sc() + self.assertEqual(resps[0], 50, 'weight 1 0') + self.assertEqual(resps[1], 50, 'weight 1 1') + + self.assertIn( + 'success', + self.conf( + { + "127.0.0.1:7081": {"weight": 3}, + "127.0.0.1:7083": {"weight": 2}, + }, + 'upstreams/one/servers', + ), + 'configure weight 2', + ) + + resps = self.get_resps_sc() + self.assertEqual(resps[0], 60, 'weight 2 0') + self.assertEqual(resps[2], 40, 'weight 2 1') + + def test_upstreams_rr_independent(self): + def sum_resps(*args): + sum = [0] * len(args[0]) + for arg in args: + sum = [x + y for x, y in zip(sum, arg)] + + return sum + + resps = self.get_resps_sc(req=30, port=7090) + self.assertEqual(resps[0], 15, 'dep two before 0') + self.assertEqual(resps[1], 15, 'dep two before 1') + + resps = self.get_resps_sc(req=30) + self.assertEqual(resps[0], 15, 'dep one before 0') + self.assertEqual(resps[1], 15, 'dep one before 1') + + self.assertIn( + 'success', + self.conf('2', 'upstreams/two/servers/127.0.0.1:7081/weight'), + 'configure dep weight', + ) + + resps = self.get_resps_sc(req=30, port=7090) + self.assertEqual(resps[0], 20, 'dep two 0') + self.assertEqual(resps[1], 10, 'dep two 1') + + resps = self.get_resps_sc(req=30) + self.assertEqual(resps[0], 15, 'dep one 0') + self.assertEqual(resps[1], 15, 'dep one 1') + + self.assertIn( + 'success', + self.conf('1', 'upstreams/two/servers/127.0.0.1:7081/weight'), + 'configure dep weight 1', + ) + + r_one, r_two = [0, 0], [0, 0] + for _ in range(10): + r_one = sum_resps(r_one, self.get_resps(req=10)) + r_two = sum_resps(r_two, self.get_resps(req=10, port=7090)) + + self.assertLessEqual( + abs(r_one[0] - r_one[1]), self.cpu_count, 'dep one mix' + ) + self.assertLessEqual( + abs(r_two[0] - r_two[1]), self.cpu_count, 'dep two mix' + ) + + def test_upstreams_rr_delay(self): + headers_delay_1 = { + 'Connection': 'close', + 'Host': 'localhost', + 'Content-Length': '0', + 'X-Delay': '1', + } + headers_no_delay = { + 'Connection': 'close', + 'Host': 'localhost', + 'Content-Length': '0', + } + + req = 50 + + socks = [] + for i in range(req): + headers = headers_delay_1 if i % 5 == 0 else headers_no_delay + _, sock = self.get( + headers=headers, + start=True, + no_recv=True, + ) + socks.append(sock) + + resps = [0, 0] + for i in range(req): + resp = self.recvall(socks[i]).decode() + socks[i].close() + + m = re.search('X-Upstream: (\d+)', resp) + resps[int(m.group(1))] += 1 + + self.assertLessEqual( + abs(resps[0] - resps[1]), self.cpu_count, 'dep two mix' + ) + + def test_upstreams_rr_active_req(self): + conns = 5 + socks = [] + socks2 = [] + + for _ in range(conns): + _, sock = self.get(start=True, no_recv=True) + socks.append(sock) + + _, sock2 = self.http( + b"""POST / HTTP/1.1 +Host: localhost +Content-Length: 10 +Connection: close + +""", + start=True, + no_recv=True, + raw=True, + ) + socks2.append(sock2) + + # Send one more request and read response to make sure that previous + # requests had enough time to reach server. + + self.assertEqual(self.get()['status'], 200) + + self.assertIn( + 'success', + self.conf( + {"127.0.0.1:7083": {"weight": 2}}, 'upstreams/one/servers', + ), + 'active req new server', + ) + self.assertIn( + 'success', + self.conf_delete('upstreams/one/servers/127.0.0.1:7083'), + 'active req server remove', + ) + self.assertIn( + 'success', self.conf_delete('listeners/*:7080'), 'delete listener' + ) + self.assertIn( + 'success', + self.conf_delete('upstreams/one'), + 'active req upstream remove', + ) + + for i in range(conns): + resp = self.recvall(socks[i]).decode() + socks[i].close() + + self.assertRegex(resp, r'X-Upstream', 'active req GET') + + resp = self.http(b"""0123456789""", sock=socks2[i], raw=True) + self.assertEqual(resp['status'], 200, 'active req POST') + + def test_upstreams_rr_bad_server(self): + self.assertIn( + 'success', + self.conf({"weight": 1}, 'upstreams/one/servers/127.0.0.1:7084'), + 'configure bad server', + ) + + resps = self.get_resps_sc(req=30) + self.assertEqual(resps[0], 10, 'bad server 0') + self.assertEqual(resps[1], 10, 'bad server 1') + self.assertEqual(sum(resps), 20, 'bad server sum') + + def test_upstreams_rr_pipeline(self): + resps = self.get_resps_sc() + + self.assertEqual(resps[0], 50, 'pipeline 0') + self.assertEqual(resps[1], 50, 'pipeline 1') + + def test_upstreams_rr_post(self): + resps = [0, 0] + for _ in range(50): + resps[ + int(self.post(body='0123456789')['headers']['X-Upstream']) + ] += 1 + resps[int(self.get()['headers']['X-Upstream'])] += 1 + + self.assertLessEqual( + abs(resps[0] - resps[1]), self.cpu_count, 'post' + ) + + def test_upstreams_rr_unix(self): + addr_0 = self.testdir + '/sock_0' + addr_1 = self.testdir + '/sock_1' + + self.assertIn( + 'success', + self.conf( + { + "*:7080": {"pass": "upstreams/one"}, + "unix:" + addr_0: {"pass": "applications/ups_0"}, + "unix:" + addr_1: {"pass": "applications/ups_1"}, + }, + 'listeners', + ), + 'configure listeners unix', + ) + + self.assertIn( + 'success', + self.conf( + {"unix:" + addr_0: {}, "unix:" + addr_1: {},}, + 'upstreams/one/servers', + ), + 'configure servers unix', + ) + + resps = self.get_resps_sc() + + self.assertEqual(resps[0], 50, 'unix 0') + self.assertEqual(resps[1], 50, 'unix 1') + + def test_upstreams_rr_ipv6(self): + self.assertIn( + 'success', + self.conf( + { + "*:7080": {"pass": "upstreams/one"}, + "[::1]:7081": {"pass": "applications/ups_0"}, + "[::1]:7082": {"pass": "applications/ups_1"}, + }, + 'listeners', + ), + 'configure listeners ipv6', + ) + + self.assertIn( + 'success', + self.conf( + {"[::1]:7081": {}, "[::1]:7082": {},}, 'upstreams/one/servers' + ), + 'configure servers ipv6', + ) + + resps = self.get_resps_sc() + + self.assertEqual(resps[0], 50, 'ipv6 0') + self.assertEqual(resps[1], 50, 'ipv6 1') + + def test_upstreams_rr_servers_empty(self): + self.assertIn( + 'success', + self.conf({}, 'upstreams/one/servers'), + 'configure servers empty', + ) + + self.assertEqual(self.get()['status'], 502, 'servers empty') + + def test_upstreams_rr_invalid(self): + self.assertIn( + 'error', self.conf({}, 'upstreams'), 'upstreams empty', + ) + self.assertIn( + 'error', self.conf({}, 'upstreams/one'), 'named upstreams empty', + ) + self.assertIn( + 'error', + self.conf({}, 'upstreams/one/servers/127.0.0.1'), + 'invalid address', + ) + self.assertIn( + 'error', + self.conf({}, 'upstreams/one/servers/127.0.0.1:7081/blah'), + 'invalid server option', + ) + self.assertIn( + 'error', + self.conf({}, 'upstreams/one/servers/127.0.0.1:7081/weight'), + 'invalid weight option', + ) + self.assertIn( + 'error', + self.conf('-1', 'upstreams/one/servers/127.0.0.1:7081/weight'), + 'invalid negative weight', + ) + + +if __name__ == '__main__': + TestUpstreamsRR.main() -- cgit From 23636ce02cf8a9abc6759b88a117f84b8457a7cd Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Thu, 12 Mar 2020 13:48:54 +0000 Subject: Tests: fixed race in USR1 signal tests. Also, minor style fixes applied. --- test/test_usr1.py | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/test/test_usr1.py b/test/test_usr1.py index dd9292c7..2b4f394b 100644 --- a/test/test_usr1.py +++ b/test/test_usr1.py @@ -10,7 +10,9 @@ class TestUSR1(TestApplicationPython): def test_usr1_access_log(self): self.load('empty') - log_path = self.testdir + '/access.log' + log = 'access.log' + log_new = 'new.log' + log_path = self.testdir + '/' + log self.assertIn( 'success', @@ -20,14 +22,12 @@ class TestUSR1(TestApplicationPython): self.assertTrue(self.waitforfiles(log_path), 'open') - log_path_new = self.testdir + '/new.log' + os.rename(log_path, self.testdir + '/' + log_new) - os.rename(log_path, log_path_new) - - self.get() + self.assertEqual(self.get()['status'], 200) self.assertIsNotNone( - self.wait_for_record(r'"GET / HTTP/1.1" 200 0 "-" "-"', 'new.log'), + self.wait_for_record(r'"GET / HTTP/1.1" 200 0 "-" "-"', log_new), 'rename new', ) self.assertFalse(os.path.isfile(log_path), 'rename old') @@ -39,32 +39,33 @@ class TestUSR1(TestApplicationPython): self.assertTrue(self.waitforfiles(log_path), 'reopen') - self.get(url='/usr1') + self.assertEqual(self.get(url='/usr1')['status'], 200) + + self.stop() self.assertIsNotNone( - self.wait_for_record( - r'"GET /usr1 HTTP/1.1" 200 0 "-" "-"', 'access.log' - ), + self.wait_for_record(r'"GET /usr1 HTTP/1.1" 200 0 "-" "-"', log), 'reopen 2', ) self.assertIsNone( - self.search_in_log(r'/usr1', 'new.log'), 'rename new 2' + self.search_in_log(r'/usr1', log_new), 'rename new 2' ) @unittest.skip('not yet') def test_usr1_unit_log(self): self.load('log_body') - log_path = self.testdir + '/unit.log' - log_path_new = self.testdir + '/new.log' + log_new = 'new.log' + log_path = self.testdir + '/' + 'unit.log' + log_path_new = self.testdir + '/' + log_new os.rename(log_path, log_path_new) body = 'body_for_a_log_new' - self.post(body=body) + self.assertEqual(self.post(body=body)['status'], 200) self.assertIsNotNone( - self.wait_for_record(body, 'new.log'), 'rename new' + self.wait_for_record(body, log_new), 'rename new' ) self.assertFalse(os.path.isfile(log_path), 'rename old') @@ -76,16 +77,18 @@ class TestUSR1(TestApplicationPython): self.assertTrue(self.waitforfiles(log_path), 'reopen') body = 'body_for_a_log_unit' - self.post(body=body) + self.assertEqual(self.post(body=body)['status'], 200) + + self.stop() self.assertIsNotNone(self.wait_for_record(body), 'rename new') - self.assertIsNone(self.search_in_log(body, 'new.log'), 'rename new 2') + self.assertIsNone(self.search_in_log(body, log_new), 'rename new 2') # merge two log files into unit.log to check alerts with open(log_path, 'w') as unit_log, \ - open(log_path_new, 'r') as new_log: - unit_log.write(new_log.read()) + open(log_path_new, 'r') as unit_log_new: + unit_log.write(unit_log_new.read()) if __name__ == '__main__': -- cgit From 2454dfe876c7d761aa46f972addd3e7c97bb8d68 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 12 Mar 2020 17:54:05 +0300 Subject: Introducing readline function in libunit. Ruby and Java modules now use this function instead of own implementations. --- src/java/nxt_jni_InputStream.c | 28 ++++------------------------ src/nxt_unit.c | 38 ++++++++++++++++++++++++++++++++++++++ src/nxt_unit.h | 3 +++ src/ruby/nxt_ruby_stream_io.c | 28 ++++++++-------------------- 4 files changed, 53 insertions(+), 44 deletions(-) diff --git a/src/java/nxt_jni_InputStream.c b/src/java/nxt_jni_InputStream.c index b96ff742..3b74b0c1 100644 --- a/src/java/nxt_jni_InputStream.c +++ b/src/java/nxt_jni_InputStream.c @@ -90,40 +90,20 @@ static jint JNICALL nxt_java_InputStream_readLine(JNIEnv *env, jclass cls, jlong req_info_ptr, jarray out, jint off, jint len) { - char *p; - jint size, b_size; uint8_t *data; ssize_t res; - nxt_unit_buf_t *b; nxt_unit_request_info_t *req; req = nxt_jlong2ptr(req_info_ptr); - size = 0; - - for (b = req->content_buf; b; b = nxt_unit_buf_next(b)) { - b_size = b->end - b->free; - p = memchr(b->free, '\n', b_size); - - if (p != NULL) { - p++; - size += p - b->free; - break; - } + data = (*env)->GetPrimitiveArrayCritical(env, out, NULL); - size += b_size; + res = nxt_unit_request_readline_size(req, len); - if (size >= len) { - break; - } + if (res > 0) { + res = nxt_unit_request_read(req, data + off, res); } - len = len < size ? len : size; - - data = (*env)->GetPrimitiveArrayCritical(env, out, NULL); - - res = nxt_unit_request_read(req, data + off, len); - nxt_unit_req_debug(req, "readLine '%.*s'", res, (char *) data + off); (*env)->ReleasePrimitiveArrayCritical(env, out, data, 0); diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 7c3d945c..07717545 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -2428,6 +2428,44 @@ nxt_unit_request_read(nxt_unit_request_info_t *req, void *dst, size_t size) } +ssize_t +nxt_unit_request_readline_size(nxt_unit_request_info_t *req, size_t max_size) +{ + char *p; + size_t l_size, b_size; + nxt_unit_buf_t *b; + + if (req->content_length == 0) { + return 0; + } + + l_size = 0; + + b = req->content_buf; + + while (b != NULL) { + b_size = b->end - b->free; + p = memchr(b->free, '\n', b_size); + + if (p != NULL) { + p++; + l_size += p - b->free; + break; + } + + l_size += b_size; + + if (max_size <= l_size) { + break; + } + + b = nxt_unit_buf_next(b); + } + + return nxt_min(max_size, l_size); +} + + static ssize_t nxt_unit_buf_read(nxt_unit_buf_t **b, uint64_t *len, void *dst, size_t size) { diff --git a/src/nxt_unit.h b/src/nxt_unit.h index c8aaa124..8e9e3015 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -335,6 +335,9 @@ int nxt_unit_response_write_cb(nxt_unit_request_info_t *req, ssize_t nxt_unit_request_read(nxt_unit_request_info_t *req, void *dst, size_t size); +ssize_t nxt_unit_request_readline_size(nxt_unit_request_info_t *req, + size_t max_size); + void nxt_unit_request_done(nxt_unit_request_info_t *req, int rc); diff --git a/src/ruby/nxt_ruby_stream_io.c b/src/ruby/nxt_ruby_stream_io.c index 7e8b3ce1..cc110035 100644 --- a/src/ruby/nxt_ruby_stream_io.c +++ b/src/ruby/nxt_ruby_stream_io.c @@ -88,9 +88,7 @@ static VALUE nxt_ruby_stream_io_gets(VALUE obj) { VALUE buf; - char *p; - size_t size, b_size; - nxt_unit_buf_t *b; + ssize_t res; nxt_ruby_run_ctx_t *run_ctx; nxt_unit_request_info_t *req; @@ -102,30 +100,20 @@ nxt_ruby_stream_io_gets(VALUE obj) return Qnil; } - size = 0; - - for (b = req->content_buf; b; b = nxt_unit_buf_next(b)) { - b_size = b->end - b->free; - p = memchr(b->free, '\n', b_size); - - if (p != NULL) { - p++; - size += p - b->free; - break; - } - - size += b_size; + res = nxt_unit_request_readline_size(req, SSIZE_MAX); + if (nxt_slow_path(res < 0)) { + return Qnil; } - buf = rb_str_buf_new(size); + buf = rb_str_buf_new(res); - if (buf == Qnil) { + if (nxt_slow_path(buf == Qnil)) { return Qnil; } - size = nxt_unit_request_read(req, RSTRING_PTR(buf), size); + res = nxt_unit_request_read(req, RSTRING_PTR(buf), res); - rb_str_set_len(buf, size); + rb_str_set_len(buf, res); return buf; } -- cgit From 7c4db34b88eab0b468425186e9b3366081e24fbb Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 12 Mar 2020 17:54:11 +0300 Subject: Python: implementing input readline and line iterator. --- src/nxt_python_wsgi.c | 147 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 2 deletions(-) diff --git a/src/nxt_python_wsgi.c b/src/nxt_python_wsgi.c index ea8b6903..14211f3f 100644 --- a/src/nxt_python_wsgi.c +++ b/src/nxt_python_wsgi.c @@ -89,8 +89,12 @@ static PyObject *nxt_py_write(PyObject *self, PyObject *args); static void nxt_py_input_dealloc(nxt_py_input_t *self); static PyObject *nxt_py_input_read(nxt_py_input_t *self, PyObject *args); static PyObject *nxt_py_input_readline(nxt_py_input_t *self, PyObject *args); +static PyObject *nxt_py_input_getline(nxt_python_run_ctx_t *ctx, size_t size); static PyObject *nxt_py_input_readlines(nxt_py_input_t *self, PyObject *args); +static PyObject *nxt_py_input_iter(PyObject *self); +static PyObject *nxt_py_input_next(PyObject *self); + static void nxt_python_print_exception(void); static int nxt_python_write(nxt_python_run_ctx_t *ctx, PyObject *bytes); @@ -142,6 +146,8 @@ static PyTypeObject nxt_py_input_type = { .tp_dealloc = (destructor) nxt_py_input_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_doc = "unit input object.", + .tp_iter = nxt_py_input_iter, + .tp_iternext = nxt_py_input_next, .tp_methods = nxt_py_input_methods, }; @@ -1229,14 +1235,151 @@ nxt_py_input_read(nxt_py_input_t *self, PyObject *args) static PyObject * nxt_py_input_readline(nxt_py_input_t *self, PyObject *args) { - return PyBytes_FromStringAndSize("", 0); + ssize_t ssize; + PyObject *obj; + Py_ssize_t n; + nxt_python_run_ctx_t *ctx; + + ctx = nxt_python_run_ctx; + if (nxt_slow_path(ctx == NULL)) { + return PyErr_Format(PyExc_RuntimeError, + "wsgi.input.readline() is called " + "outside of WSGI request processing"); + } + + n = PyTuple_GET_SIZE(args); + + if (n > 0) { + if (n != 1) { + return PyErr_Format(PyExc_TypeError, "invalid number of arguments"); + } + + obj = PyTuple_GET_ITEM(args, 0); + + ssize = PyNumber_AsSsize_t(obj, PyExc_OverflowError); + + if (nxt_fast_path(ssize > 0)) { + return nxt_py_input_getline(ctx, ssize); + } + + if (ssize == 0) { + return PyBytes_FromStringAndSize("", 0); + } + + if (ssize != -1) { + return PyErr_Format(PyExc_ValueError, + "the read line size cannot be zero or less"); + } + + if (PyErr_Occurred()) { + return NULL; + } + } + + return nxt_py_input_getline(ctx, SSIZE_MAX); +} + + +static PyObject * +nxt_py_input_getline(nxt_python_run_ctx_t *ctx, size_t size) +{ + void *buf; + ssize_t res; + PyObject *content; + + res = nxt_unit_request_readline_size(ctx->req, size); + if (nxt_slow_path(res < 0)) { + return NULL; + } + + if (res == 0) { + return PyBytes_FromStringAndSize("", 0); + } + + content = PyBytes_FromStringAndSize(NULL, res); + if (nxt_slow_path(content == NULL)) { + return NULL; + } + + buf = PyBytes_AS_STRING(content); + + res = nxt_unit_request_read(ctx->req, buf, res); + + return content; } static PyObject * nxt_py_input_readlines(nxt_py_input_t *self, PyObject *args) { - return PyList_New(0); + PyObject *res; + nxt_python_run_ctx_t *ctx; + + ctx = nxt_python_run_ctx; + if (nxt_slow_path(ctx == NULL)) { + return PyErr_Format(PyExc_RuntimeError, + "wsgi.input.readlines() is called " + "outside of WSGI request processing"); + } + + res = PyList_New(0); + if (nxt_slow_path(res == NULL)) { + return NULL; + } + + for ( ;; ) { + PyObject *line = nxt_py_input_getline(ctx, SSIZE_MAX); + if (nxt_slow_path(line == NULL)) { + Py_DECREF(res); + return NULL; + } + + if (PyBytes_GET_SIZE(line) == 0) { + Py_DECREF(line); + return res; + } + + PyList_Append(res, line); + Py_DECREF(line); + } + + return res; +} + + +static PyObject * +nxt_py_input_iter(PyObject *self) +{ + Py_INCREF(self); + return self; +} + + +static PyObject * +nxt_py_input_next(PyObject *self) +{ + PyObject *line; + nxt_python_run_ctx_t *ctx; + + ctx = nxt_python_run_ctx; + if (nxt_slow_path(ctx == NULL)) { + return PyErr_Format(PyExc_RuntimeError, + "wsgi.input.next() is called " + "outside of WSGI request processing"); + } + + line = nxt_py_input_getline(ctx, SSIZE_MAX); + if (nxt_slow_path(line == NULL)) { + return NULL; + } + + if (PyBytes_GET_SIZE(line) == 0) { + Py_DECREF(line); + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + + return line; } -- cgit From f3e6726098220701dc2193c440852d04508cf972 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 12 Mar 2020 17:54:15 +0300 Subject: Tests: added Python input readline and iterator tests. --- test/python/input_iter/wsgi.py | 17 ++++++-- test/python/input_readline/wsgi.py | 20 +++++++++ test/python/input_readline_size/wsgi.py | 16 ++++++++ test/python/input_readlines/wsgi.py | 5 +++ test/test_python_application.py | 73 +++++++++++++++++++++++++++++++-- 5 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 test/python/input_readline/wsgi.py create mode 100644 test/python/input_readline_size/wsgi.py create mode 100644 test/python/input_readlines/wsgi.py diff --git a/test/python/input_iter/wsgi.py b/test/python/input_iter/wsgi.py index d3bf437f..04a6a06c 100644 --- a/test/python/input_iter/wsgi.py +++ b/test/python/input_iter/wsgi.py @@ -1,5 +1,16 @@ def application(environ, start_response): - body = bytes(environ['wsgi.input'].__iter__()) + body = [] + content_length = 0 - start_response('200', [('Content-Length', str(len(body)))]) - return [body] + for l in environ['wsgi.input'].__iter__(): + body.append(l) + content_length += len(l) + + start_response( + '200', + [ + ('Content-Length', str(content_length)), + ('X-Lines-Count', str(len(body))), + ], + ) + return body diff --git a/test/python/input_readline/wsgi.py b/test/python/input_readline/wsgi.py new file mode 100644 index 00000000..ec42e0c8 --- /dev/null +++ b/test/python/input_readline/wsgi.py @@ -0,0 +1,20 @@ +def application(environ, start_response): + body = [] + content_length = 0 + + while True: + l = environ['wsgi.input'].readline() + if not l: + break + + body.append(l) + content_length += len(l) + + start_response( + '200', + [ + ('Content-Length', str(content_length)), + ('X-Lines-Count', str(len(body))), + ], + ) + return body diff --git a/test/python/input_readline_size/wsgi.py b/test/python/input_readline_size/wsgi.py new file mode 100644 index 00000000..36cf07b0 --- /dev/null +++ b/test/python/input_readline_size/wsgi.py @@ -0,0 +1,16 @@ +def application(environ, start_response): + body = [] + + while True: + l = environ['wsgi.input'].readline(9) + if not l: + break + + body.append(l) + + if len(l) > 9: + body.append(b'len(l) > 9: ' + l) + break + + start_response('200', [('X-Lines-Count', str(len(body)))]) + return body diff --git a/test/python/input_readlines/wsgi.py b/test/python/input_readlines/wsgi.py new file mode 100644 index 00000000..64b03d79 --- /dev/null +++ b/test/python/input_readlines/wsgi.py @@ -0,0 +1,5 @@ +def application(environ, start_response): + body = environ['wsgi.input'].readlines() + + start_response('200', [('X-Lines-Count', str(len(body)))]) + return body diff --git a/test/test_python_application.py b/test/test_python_application.py index 818816d0..460cc804 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -384,13 +384,80 @@ Connection: close self.assertEqual(self.get()['status'], 500, 'start response exit') - @unittest.skip('not yet') def test_python_application_input_iter(self): self.load('input_iter') - body = '0123456789' + body = '''0123456789 +next line + +last line''' + + resp = self.post(body=body) + self.assertEqual(resp['body'], body, 'input iter') + self.assertEqual( + resp['headers']['X-Lines-Count'], '4', 'input iter lines' + ) + + def test_python_application_input_readline(self): + self.load('input_readline') + + body = '''0123456789 +next line + +last line''' + + resp = self.post(body=body) + self.assertEqual(resp['body'], body, 'input readline') + self.assertEqual( + resp['headers']['X-Lines-Count'], '4', 'input readline lines' + ) + + def test_python_application_input_readline_size(self): + self.load('input_readline_size') + + body = '''0123456789 +next line + +last line''' + + self.assertEqual( + self.post(body=body)['body'], body, 'input readline size' + ) + self.assertEqual( + self.post(body='0123')['body'], '0123', 'input readline size less' + ) + + def test_python_application_input_readlines(self): + self.load('input_readlines') + + body = '''0123456789 +next line + +last line''' + + resp = self.post(body=body) + self.assertEqual(resp['body'], body, 'input readlines') + self.assertEqual( + resp['headers']['X-Lines-Count'], '4', 'input readlines lines' + ) + + def test_python_application_input_readlines_huge(self): + self.load('input_readlines') + + body = ( + '''0123456789 abcdefghi +next line: 0123456789 abcdefghi - self.assertEqual(self.post(body=body)['body'], body, 'input iter') +last line: 987654321 +''' + * 512 + ) + + self.assertEqual( + self.post(body=body, read_buffer_size=16384)['body'], + body, + 'input readlines huge', + ) def test_python_application_input_read_length(self): self.load('input_read_length') -- cgit From 0b5aabfc3f6dafa00d4cd4da595bceeefd1a1d27 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 12 Mar 2020 17:54:19 +0300 Subject: Checking Content-Length value right after header parse. The check was moved from the request body read stage. --- src/nxt_h1proto.c | 5 ----- src/nxt_http_request.c | 9 ++++++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 1a6c537e..c326ef30 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -847,11 +847,6 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) goto ready; } - if (r->content_length_n > (nxt_off_t) r->conf->socket_conf->max_body_size) { - status = NXT_HTTP_PAYLOAD_TOO_LARGE; - goto error; - } - body_length = (size_t) r->content_length_n; b = r->body; diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index 14c75dab..d610f65d 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -186,7 +186,7 @@ nxt_int_t nxt_http_request_content_length(void *ctx, nxt_http_field_t *field, uintptr_t data) { - nxt_off_t n; + nxt_off_t n, max_body_size; nxt_http_request_t *r; r = ctx; @@ -198,6 +198,13 @@ nxt_http_request_content_length(void *ctx, nxt_http_field_t *field, if (nxt_fast_path(n >= 0)) { r->content_length_n = n; + + max_body_size = r->conf->socket_conf->max_body_size; + + if (nxt_slow_path(n > max_body_size)) { + return NXT_HTTP_PAYLOAD_TOO_LARGE; + } + return NXT_OK; } } -- cgit From 08b65721e25b1b94affc12078a623a11341525d1 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 12 Mar 2020 17:54:24 +0300 Subject: Moving request memory pool retain call after RPC data allocation. If the call is done only after a successful RPC data allocation, its corresponding release call is not missed, which avoids a potential leak. --- src/nxt_http_request.c | 10 ---------- src/nxt_router.c | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index d610f65d..51553f0c 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -326,18 +326,8 @@ nxt_http_action_t * nxt_http_application_handler(nxt_task_t *task, nxt_http_request_t *r, nxt_http_action_t *action) { - nxt_event_engine_t *engine; - nxt_debug(task, "http application handler"); - nxt_mp_retain(r->mem_pool); - - engine = task->thread->engine; - r->timer.task = &engine->task; - r->timer.work_queue = &engine->fast_work_queue; - r->timer.log = engine->task.log; - r->timer.bias = NXT_TIMER_DEFAULT_BIAS; - /* * TODO: need an application flag to get local address * required by "SERVER_ADDR" in Pyhton and PHP. Not used in Go. diff --git a/src/nxt_router.c b/src/nxt_router.c index d77ffa2b..9138a9a3 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -4705,6 +4705,21 @@ nxt_router_process_http_request(nxt_task_t *task, nxt_http_request_t *r, return; } + /* + * At this point we have request req_rpc_data allocated and registered + * in port handlers. Need to fixup request memory pool. Counterpart + * release will be called via following call chain: + * nxt_request_rpc_data_unlink() -> + * nxt_router_http_request_done() -> + * nxt_router_http_request_release() + */ + nxt_mp_retain(r->mem_pool); + + r->timer.task = &engine->task; + r->timer.work_queue = &engine->fast_work_queue; + r->timer.log = engine->task.log; + r->timer.bias = NXT_TIMER_DEFAULT_BIAS; + req_rpc_data->stream = nxt_port_rpc_ex_stream(req_rpc_data); req_rpc_data->app = app; -- cgit From 5296be0b82784eb90abc86339e6c16841e9a9727 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 12 Mar 2020 17:54:29 +0300 Subject: Using disk file to store large request body. This closes #386 on GitHub. --- auto/help | 1 + auto/options | 6 ++ auto/save | 1 + auto/summary | 1 + configure | 2 + pkg/deb/Makefile | 1 + pkg/rpm/Makefile | 1 + src/nxt_conf_validation.c | 10 +++ src/nxt_conn_write.c | 96 +++++++++++++++++++++++++ src/nxt_h1proto.c | 178 +++++++++++++++++++++++++++++++++++++++------- src/nxt_http_request.c | 8 +++ src/nxt_router.c | 37 +++++++++- src/nxt_router.h | 2 + src/nxt_runtime.c | 18 +++++ src/nxt_runtime.h | 1 + src/nxt_unit.c | 126 ++++++++++++++++++++++++++++++-- src/nxt_unit.h | 1 + test/unit/main.py | 1 + 18 files changed, 455 insertions(+), 36 deletions(-) diff --git a/auto/help b/auto/help index fe0c7056..f5f10010 100644 --- a/auto/help +++ b/auto/help @@ -20,6 +20,7 @@ cat << END --incdir=DIRECTORY set includes directory name, default: "$NXT_INCDIR" --modules=DIRECTORY set modules directory name, default: "$NXT_MODULES" --state=DIRECTORY set state directory name, default: "$NXT_STATE" + --tmp=DIRECTORY set tmp directory name, default: "$NXT_TMP" --pid=FILE set pid filename, default: "$NXT_PID" --log=FILE set log filename, default: "$NXT_LOG" diff --git a/auto/options b/auto/options index 0d31abad..d315b227 100644 --- a/auto/options +++ b/auto/options @@ -58,6 +58,7 @@ do --incdir=*) NXT_INCDIR="$value" ;; --modules=*) NXT_MODULES="$value" ;; --state=*) NXT_STATE="$value" ;; + --tmp=*) NXT_TMP="$value" ;; --pid=*) NXT_PID="$value" ;; --log=*) NXT_LOG="$value" ;; @@ -149,6 +150,11 @@ case "$NXT_STATE" in *) NXT_STATE="$NXT_PREFIX$NXT_STATE" ;; esac +case "$NXT_TMP" in + /*) ;; + *) NXT_TMP="$NXT_PREFIX$NXT_TMP" ;; +esac + case "$NXT_PID" in /*) ;; *) NXT_PID="$NXT_PREFIX$NXT_PID" ;; diff --git a/auto/save b/auto/save index 350c9c1f..19ef09ec 100644 --- a/auto/save +++ b/auto/save @@ -29,5 +29,6 @@ NXT_LIB_AUX_LIBS= NXT_LIB_UNIT_STATIC='$NXT_LIB_UNIT_STATIC' NXT_MODULES='$NXT_MODULES' +NXT_TMP='$NXT_TMP' END diff --git a/auto/summary b/auto/summary index 59267f6c..833d20c0 100644 --- a/auto/summary +++ b/auto/summary @@ -13,6 +13,7 @@ Unit configuration summary: include directory: ......... "$NXT_INCDIR" modules directory: ......... "$NXT_MODULES" state directory: ........... "$NXT_STATE" + tmp directory: ............. "$NXT_TMP" pid file: .................. "$NXT_PID" log file: .................. "$NXT_LOG" diff --git a/configure b/configure index b6cd3087..c67e4728 100755 --- a/configure +++ b/configure @@ -37,6 +37,7 @@ NXT_LIBDIR="lib" NXT_INCDIR="include" NXT_MODULES="modules" NXT_STATE="state" +NXT_TMP="tmp" NXT_PID="unit.pid" NXT_LOG="unit.log" NXT_CONTROL="unix:control.unit.sock" @@ -86,6 +87,7 @@ cat << END >> $NXT_AUTO_CONFIG_H #define NXT_LOG "$NXT_LOG" #define NXT_MODULES "$NXT_MODULES" #define NXT_STATE "$NXT_STATE" +#define NXT_TMP "$NXT_TMP" #define NXT_CONTROL_SOCK "$NXT_CONTROL" diff --git a/pkg/deb/Makefile b/pkg/deb/Makefile index 13063fd8..797ff438 100644 --- a/pkg/deb/Makefile +++ b/pkg/deb/Makefile @@ -129,6 +129,7 @@ CONFIGURE_ARGS=\ --control="unix:/var/run/control.unit.sock" \ --pid=/var/run/unit.pid \ --log=/var/log/unit.log \ + --tmp=/var/tmp \ --tests \ --openssl diff --git a/pkg/rpm/Makefile b/pkg/rpm/Makefile index 34f79bcc..8bc96d99 100644 --- a/pkg/rpm/Makefile +++ b/pkg/rpm/Makefile @@ -144,6 +144,7 @@ CONFIGURE_ARGS=\ --control="unix:/var/run/unit/control.sock" \ --pid=/var/run/unit/unit.pid \ --log=/var/log/unit/unit.log \ + --tmp=/var/tmp \ --tests \ --openssl diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 86c1dbcb..3a3654bd 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -182,11 +182,21 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { NULL, NULL }, + { nxt_string("body_buffer_size"), + NXT_CONF_VLDT_INTEGER, + NULL, + NULL }, + { nxt_string("max_body_size"), NXT_CONF_VLDT_INTEGER, NULL, NULL }, + { nxt_string("body_temp_path"), + NXT_CONF_VLDT_STRING, + NULL, + NULL }, + { nxt_string("websocket"), NXT_CONF_VLDT_OBJECT, &nxt_conf_vldt_object, diff --git a/src/nxt_conn_write.c b/src/nxt_conn_write.c index 298d8f75..d7a6a8da 100644 --- a/src/nxt_conn_write.c +++ b/src/nxt_conn_write.c @@ -9,6 +9,8 @@ static void nxt_conn_write_timer_handler(nxt_task_t *task, void *obj, void *data); +static ssize_t nxt_conn_io_sendfile(nxt_task_t *task, nxt_sendbuf_t *sb); +static ssize_t nxt_sendfile(int fd, int s, off_t pos, size_t size); void @@ -170,10 +172,104 @@ nxt_conn_io_sendbuf(nxt_task_t *task, nxt_sendbuf_t *sb) return 0; } + if (niov == 0 && nxt_buf_is_file(sb->buf)) { + return nxt_conn_io_sendfile(task, sb); + } + return nxt_conn_io_writev(task, sb, iov, niov); } +static ssize_t +nxt_conn_io_sendfile(nxt_task_t *task, nxt_sendbuf_t *sb) +{ + size_t size; + ssize_t n; + nxt_buf_t *b; + nxt_err_t err; + + b = sb->buf; + + for ( ;; ) { + size = b->file_end - b->file_pos; + + n = nxt_sendfile(b->file->fd, sb->socket, b->file_pos, size); + + err = (n == -1) ? nxt_errno : 0; + + nxt_debug(task, "sendfile(%FD, %d, @%O, %uz): %z", + b->file->fd, sb->socket, b->file_pos, size, n); + + if (n > 0) { + if (n < (ssize_t) size) { + sb->ready = 0; + } + + return n; + } + + if (nxt_slow_path(n == 0)) { + nxt_alert(task, "sendfile() reported that file was truncated at %O", + b->file_pos); + + return NXT_ERROR; + } + + /* n == -1 */ + + switch (err) { + + case NXT_EAGAIN: + sb->ready = 0; + nxt_debug(task, "sendfile() %E", err); + + return NXT_AGAIN; + + case NXT_EINTR: + nxt_debug(task, "sendfile() %E", err); + continue; + + default: + sb->error = err; + nxt_log(task, nxt_socket_error_level(err), + "sendfile(%FD, %d, @%O, %uz) failed %E", + b->file->fd, sb->socket, b->file_pos, size, err); + + return NXT_ERROR; + } + } +} + + +static ssize_t +nxt_sendfile(int fd, int s, off_t pos, size_t size) +{ + ssize_t res; + +#ifdef NXT_HAVE_MACOSX_SENDFILE + off_t sent = size; + + int rc = sendfile(fd, s, pos, &sent, NULL, 0); + + res = (rc == 0 || sent > 0) ? sent : -1; +#endif + +#ifdef NXT_HAVE_FREEBSD_SENDFILE + off_t sent = 0; + + int rc = sendfile(fd, s, pos, size, NULL, &sent, 0); + + res = (rc == 0 || sent > 0) ? sent : -1; +#endif + +#ifdef NXT_HAVE_LINUX_SENDFILE + res = sendfile(s, fd, &pos, size); +#endif + + return res; +} + + ssize_t nxt_conn_io_writev(nxt_task_t *task, nxt_sendbuf_t *sb, struct iovec *iov, nxt_uint_t niov) diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index c326ef30..35918bd8 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -817,12 +817,16 @@ nxt_h1p_transfer_encoding(void *ctx, nxt_http_field_t *field, uintptr_t data) static void nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) { - size_t size, body_length; + size_t size, body_length, body_buffer_size, body_rest; + ssize_t res; + nxt_str_t *tmp_path, tmp_name; nxt_buf_t *in, *b; nxt_conn_t *c; nxt_h1proto_t *h1p; nxt_http_status_t status; + static const nxt_str_t tmp_name_pattern = nxt_string("/req-XXXXXXXX"); + h1p = r->proto.h1; nxt_debug(task, "h1p request body read %O te:%d", @@ -849,36 +853,95 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) body_length = (size_t) r->content_length_n; - b = r->body; + body_buffer_size = nxt_min(r->conf->socket_conf->body_buffer_size, + body_length); + + if (body_length > body_buffer_size) { + tmp_path = &r->conf->socket_conf->body_temp_path; + + tmp_name.length = tmp_path->length + tmp_name_pattern.length; + + b = nxt_buf_file_alloc(r->mem_pool, + body_buffer_size + sizeof(nxt_file_t) + + tmp_name.length + 1, 0); + + } else { + /* This initialization required for CentOS 6, gcc 4.4.7. */ + tmp_path = NULL; + tmp_name.length = 0; + + b = nxt_buf_mem_alloc(r->mem_pool, body_buffer_size, 0); + } + + if (nxt_slow_path(b == NULL)) { + status = NXT_HTTP_INTERNAL_SERVER_ERROR; + goto error; + } + + r->body = b; + + if (body_length > body_buffer_size) { + tmp_name.start = nxt_pointer_to(b->mem.start, sizeof(nxt_file_t)); + + memcpy(tmp_name.start, tmp_path->start, tmp_path->length); + memcpy(tmp_name.start + tmp_path->length, tmp_name_pattern.start, + tmp_name_pattern.length); + tmp_name.start[tmp_name.length] = '\0'; + + b->file = (nxt_file_t *) b->mem.start; + nxt_memzero(b->file, sizeof(nxt_file_t)); + b->file->fd = -1; + b->file->size = body_length; + + b->mem.start += sizeof(nxt_file_t) + tmp_name.length + 1; + b->mem.pos = b->mem.start; + b->mem.free = b->mem.start; + + b->file->fd = mkstemp((char *) tmp_name.start); + if (nxt_slow_path(b->file->fd == -1)) { + nxt_log(task, NXT_LOG_ERR, "mkstemp() failed %E", nxt_errno); - if (b == NULL) { - b = nxt_buf_mem_alloc(r->mem_pool, body_length, 0); - if (nxt_slow_path(b == NULL)) { status = NXT_HTTP_INTERNAL_SERVER_ERROR; goto error; } - r->body = b; + nxt_debug(task, "create body tmp file \"%V\", %d", + &tmp_name, b->file->fd); + + unlink((char *) tmp_name.start); } + body_rest = body_length; + in = h1p->conn->read; size = nxt_buf_mem_used_size(&in->mem); if (size != 0) { - if (size > body_length) { - size = body_length; + size = nxt_min(size, body_length); + + if (nxt_buf_is_file(b)) { + res = nxt_fd_write(b->file->fd, in->mem.pos, size); + if (nxt_slow_path(res < (ssize_t) size)) { + status = NXT_HTTP_INTERNAL_SERVER_ERROR; + goto error; + } + + b->file_end += size; + + } else { + size = nxt_min(body_buffer_size, size); + b->mem.free = nxt_cpymem(b->mem.free, in->mem.pos, size); + body_buffer_size -= size; } - b->mem.free = nxt_cpymem(b->mem.free, in->mem.pos, size); in->mem.pos += size; + body_rest -= size; } - size = nxt_buf_mem_free_size(&b->mem); - - nxt_debug(task, "h1p body rest: %uz", size); + nxt_debug(task, "h1p body rest: %uz", body_rest); - if (size != 0) { + if (body_rest != 0) { in->next = h1p->buffers; h1p->buffers = in; h1p->nbuffers++; @@ -891,6 +954,13 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) return; } + if (nxt_buf_is_file(b)) { + b->mem.start = NULL; + b->mem.end = NULL; + b->mem.pos = NULL; + b->mem.free = NULL; + } + ready: r->state->ready_handler(task, r, NULL); @@ -922,7 +992,9 @@ static const nxt_conn_state_t nxt_h1p_read_body_state static void nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data) { - size_t size; + size_t size, body_rest; + ssize_t res; + nxt_buf_t *b; nxt_conn_t *c; nxt_h1proto_t *h1p; nxt_http_request_t *r; @@ -933,18 +1005,59 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data) nxt_debug(task, "h1p conn request body read"); - size = nxt_buf_mem_free_size(&c->read->mem); - - nxt_debug(task, "h1p body rest: %uz", size); + r = h1p->request; engine = task->thread->engine; - if (size != 0) { + b = c->read; + + if (nxt_buf_is_file(b)) { + body_rest = b->file->size - b->file_end; + + size = nxt_buf_mem_used_size(&b->mem); + size = nxt_min(size, body_rest); + + res = nxt_fd_write(b->file->fd, b->mem.pos, size); + if (nxt_slow_path(res < (ssize_t) size)) { + nxt_h1p_request_error(task, h1p, r); + return; + } + + b->file_end += size; + body_rest -= res; + + b->mem.pos += size; + + if (b->mem.pos == b->mem.free) { + if (body_rest >= (size_t) nxt_buf_mem_size(&b->mem)) { + b->mem.free = b->mem.start; + + } else { + /* This required to avoid reading next request. */ + b->mem.free = b->mem.end - body_rest; + } + + b->mem.pos = b->mem.free; + } + + } else { + body_rest = nxt_buf_mem_free_size(&c->read->mem); + } + + nxt_debug(task, "h1p body rest: %uz", body_rest); + + if (body_rest != 0) { nxt_conn_read(engine, c); } else { + if (nxt_buf_is_file(b)) { + b->mem.start = NULL; + b->mem.end = NULL; + b->mem.pos = NULL; + b->mem.free = NULL; + } + c->read = NULL; - r = h1p->request; r->state->ready_handler(task, r, NULL); } @@ -2140,7 +2253,13 @@ nxt_h1p_peer_header_send(nxt_task_t *task, nxt_http_peer_t *peer) c->write_state = &nxt_h1p_peer_header_send_state; if (r->body != NULL) { - body = nxt_buf_mem_alloc(r->mem_pool, 0, 0); + if (nxt_buf_is_file(r->body)) { + body = nxt_buf_file_alloc(r->mem_pool, 0, 0); + + } else { + body = nxt_buf_mem_alloc(r->mem_pool, 0, 0); + } + if (nxt_slow_path(body == NULL)) { r->state->error_handler(task, r, peer); return; @@ -2148,8 +2267,15 @@ nxt_h1p_peer_header_send(nxt_task_t *task, nxt_http_peer_t *peer) header->next = body; - body->mem = r->body->mem; - size += nxt_buf_mem_used_size(&body->mem); + if (nxt_buf_is_file(r->body)) { + body->file = r->body->file; + body->file_end = r->body->file_end; + + } else { + body->mem = r->body->mem; + } + + size += nxt_buf_used_size(body); // nxt_mp_retain(r->mem_pool); } @@ -2205,13 +2331,13 @@ nxt_h1p_peer_header_sent(nxt_task_t *task, void *obj, void *data) c->write = nxt_sendbuf_completion(task, &engine->fast_work_queue, c->write); - if (c->write == NULL) { - r = peer->request; - r->state->ready_handler(task, r, peer); + if (c->write != NULL) { + nxt_conn_write(engine, c); return; } - nxt_conn_write(engine, c); + r = peer->request; + r->state->ready_handler(task, r, peer); } diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index 51553f0c..72aaa290 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -569,6 +569,14 @@ nxt_http_request_close_handler(nxt_task_t *task, void *obj, void *data) r->proto.any = NULL; + if (r->body != NULL && nxt_buf_is_file(r->body) + && r->body->file->fd != -1) + { + nxt_fd_close(r->body->file->fd); + + r->body->file->fd = -1; + } + if (nxt_fast_path(proto.any != NULL)) { protocol = r->protocol; diff --git a/src/nxt_router.c b/src/nxt_router.c index 9138a9a3..a913284c 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1361,6 +1361,12 @@ static nxt_conf_map_t nxt_router_http_conf[] = { NXT_CONF_MAP_MSEC, offsetof(nxt_socket_conf_t, send_timeout), }, + + { + nxt_string("body_temp_path"), + NXT_CONF_MAP_STR, + offsetof(nxt_socket_conf_t, body_temp_path), + }, }; @@ -1397,6 +1403,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_int_t ret; nxt_str_t name, path; nxt_app_t *app, *prev; + nxt_str_t *t; nxt_router_t *router; nxt_app_joint_t *app_joint; nxt_conf_value_t *conf, *http, *value, *websocket; @@ -1698,6 +1705,8 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, skcf->websocket_conf.read_timeout = 60 * 1000; skcf->websocket_conf.keepalive_interval = 30 * 1000; + nxt_str_null(&skcf->body_temp_path); + if (http != NULL) { ret = nxt_conf_map_object(mp, http, nxt_router_http_conf, nxt_nitems(nxt_router_http_conf), @@ -1719,6 +1728,13 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, } } + t = &skcf->body_temp_path; + + if (t->length == 0) { + t->start = (u_char *) task->thread->runtime->tmp; + t->length = nxt_strlen(t->start); + } + #if (NXT_TLS) value = nxt_conf_get_path(listener, &certificate_path); @@ -4758,7 +4774,8 @@ static void nxt_router_app_prepare_request(nxt_task_t *task, nxt_request_app_link_t *req_app_link) { - nxt_buf_t *buf; + nxt_fd_t fd; + nxt_buf_t *buf, *body; nxt_int_t res; nxt_port_t *port, *c_port, *reply_port; nxt_apr_action_t apr_action; @@ -4817,8 +4834,14 @@ nxt_router_app_prepare_request(nxt_task_t *task, goto release_port; } - res = nxt_port_socket_twrite(task, port, NXT_PORT_MSG_REQ_HEADERS, - -1, req_app_link->stream, reply_port->id, buf, + body = req_app_link->request->body; + fd = (body != NULL && nxt_buf_is_file(body)) ? body->file->fd : -1; + + res = nxt_port_socket_twrite(task, port, + NXT_PORT_MSG_REQ_HEADERS + | NXT_PORT_MSG_CLOSE_FD, + fd, + req_app_link->stream, reply_port->id, buf, &req_app_link->msg_info.tracking); if (nxt_slow_path(res != NXT_OK)) { @@ -4827,6 +4850,10 @@ nxt_router_app_prepare_request(nxt_task_t *task, goto release_port; } + if (fd != -1) { + body->file->fd = -1; + } + release_port: nxt_router_app_port_release(task, port, apr_action); @@ -5151,6 +5178,10 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, } } + if (r->body != NULL && nxt_buf_is_file(r->body)) { + lseek(r->body->file->fd, 0, SEEK_SET); + } + return out; } diff --git a/src/nxt_router.h b/src/nxt_router.h index 85ef9a6c..08142ce3 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -187,6 +187,8 @@ typedef struct { nxt_websocket_conf_t websocket_conf; + nxt_str_t body_temp_path; + #if (NXT_TLS) nxt_tls_conf_t *tls; #endif diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index 80b25c1b..f6d80ccb 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -693,6 +693,7 @@ nxt_runtime_conf_init(nxt_task_t *task, nxt_runtime_t *rt) rt->modules = NXT_MODULES; rt->state = NXT_STATE; rt->control = NXT_CONTROL_SOCK; + rt->tmp = NXT_TMP; nxt_memzero(&rt->capabilities, sizeof(nxt_capabilities_t)); @@ -835,6 +836,7 @@ nxt_runtime_conf_read_cmd(nxt_task_t *task, nxt_runtime_t *rt) static const char no_modules[] = "option \"--modules\" requires directory\n"; static const char no_state[] = "option \"--state\" requires directory\n"; + static const char no_tmp[] = "option \"--tmp\" requires directory\n"; static const char help[] = "\n" @@ -859,6 +861,9 @@ nxt_runtime_conf_read_cmd(nxt_task_t *task, nxt_runtime_t *rt) " --state DIRECTORY set state directory name\n" " default: \"" NXT_STATE "\"\n" "\n" + " --tmp DIRECTORY set tmp directory name\n" + " default: \"" NXT_TMP "\"\n" + "\n" " --user USER set non-privileged processes to run" " as specified user\n" " default: \"" NXT_USER "\"\n" @@ -966,6 +971,19 @@ nxt_runtime_conf_read_cmd(nxt_task_t *task, nxt_runtime_t *rt) continue; } + if (nxt_strcmp(p, "--tmp") == 0) { + if (*argv == NULL) { + write(STDERR_FILENO, no_tmp, nxt_length(no_tmp)); + return NXT_ERROR; + } + + p = *argv++; + + rt->tmp = p; + + continue; + } + if (nxt_strcmp(p, "--no-daemon") == 0) { rt->daemon = 0; continue; diff --git a/src/nxt_runtime.h b/src/nxt_runtime.h index f8d19ec6..a364c38c 100644 --- a/src/nxt_runtime.h +++ b/src/nxt_runtime.h @@ -68,6 +68,7 @@ struct nxt_runtime_s { const char *conf; const char *conf_tmp; const char *control; + const char *tmp; nxt_str_t certs; diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 07717545..7a4124fb 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -76,6 +76,8 @@ static nxt_unit_read_buf_t *nxt_unit_read_buf_get_impl( nxt_unit_ctx_impl_t *ctx_impl); static void nxt_unit_read_buf_release(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf); +static nxt_unit_mmap_buf_t *nxt_unit_request_preread( + nxt_unit_request_info_t *req, size_t size); static ssize_t nxt_unit_buf_read(nxt_unit_buf_t **b, uint64_t *len, void *dst, size_t size); static nxt_port_mmap_header_t *nxt_unit_mmap_get(nxt_unit_ctx_t *ctx, @@ -961,6 +963,9 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) req_impl->incoming_buf->prev = &req_impl->incoming_buf; recv_msg->incoming_buf = NULL; + req->content_fd = recv_msg->fd; + recv_msg->fd = -1; + req->response_max_fields = 0; req_impl->state = NXT_UNIT_RS_START; req_impl->websocket = 0; @@ -1178,6 +1183,12 @@ nxt_unit_request_info_release(nxt_unit_request_info_t *req) nxt_unit_mmap_buf_free(req_impl->incoming_buf); } + if (req->content_fd != -1) { + close(req->content_fd); + + req->content_fd = -1; + } + /* * Process release should go after buffers release to guarantee mmap * existence. @@ -2423,17 +2434,46 @@ nxt_unit_response_write_cb(nxt_unit_request_info_t *req, ssize_t nxt_unit_request_read(nxt_unit_request_info_t *req, void *dst, size_t size) { - return nxt_unit_buf_read(&req->content_buf, &req->content_length, - dst, size); + ssize_t buf_res, res; + + buf_res = nxt_unit_buf_read(&req->content_buf, &req->content_length, + dst, size); + + if (buf_res < (ssize_t) size && req->content_fd != -1) { + res = read(req->content_fd, dst, size); + if (res < 0) { + nxt_unit_req_alert(req, "failed to read content: %s (%d)", + strerror(errno), errno); + + return res; + } + + if (res < (ssize_t) size) { + close(req->content_fd); + + req->content_fd = -1; + } + + req->content_length -= res; + size -= res; + + dst = nxt_pointer_to(dst, res); + + } else { + res = 0; + } + + return buf_res + res; } ssize_t nxt_unit_request_readline_size(nxt_unit_request_info_t *req, size_t max_size) { - char *p; - size_t l_size, b_size; - nxt_unit_buf_t *b; + char *p; + size_t l_size, b_size; + nxt_unit_buf_t *b; + nxt_unit_mmap_buf_t *mmap_buf, *preread_buf; if (req->content_length == 0) { return 0; @@ -2459,6 +2499,19 @@ nxt_unit_request_readline_size(nxt_unit_request_info_t *req, size_t max_size) break; } + mmap_buf = nxt_container_of(b, nxt_unit_mmap_buf_t, buf); + if (mmap_buf->next == NULL + && req->content_fd != -1 + && l_size < req->content_length) + { + preread_buf = nxt_unit_request_preread(req, 16384); + if (nxt_slow_path(preread_buf == NULL)) { + return -1; + } + + nxt_unit_mmap_buf_insert(&mmap_buf->next, preread_buf); + } + b = nxt_unit_buf_next(b); } @@ -2466,19 +2519,78 @@ nxt_unit_request_readline_size(nxt_unit_request_info_t *req, size_t max_size) } +static nxt_unit_mmap_buf_t * +nxt_unit_request_preread(nxt_unit_request_info_t *req, size_t size) +{ + ssize_t res; + nxt_unit_mmap_buf_t *mmap_buf; + + if (req->content_fd == -1) { + nxt_unit_req_alert(req, "preread: content_fd == -1"); + return NULL; + } + + mmap_buf = nxt_unit_mmap_buf_get(req->ctx); + if (nxt_slow_path(mmap_buf == NULL)) { + nxt_unit_req_alert(req, "preread: failed to allocate buf"); + return NULL; + } + + mmap_buf->free_ptr = malloc(size); + if (nxt_slow_path(mmap_buf->free_ptr == NULL)) { + nxt_unit_req_alert(req, "preread: failed to allocate buf memory"); + nxt_unit_mmap_buf_release(mmap_buf); + return NULL; + } + + mmap_buf->plain_ptr = mmap_buf->free_ptr; + + mmap_buf->hdr = NULL; + mmap_buf->buf.start = mmap_buf->free_ptr; + mmap_buf->buf.free = mmap_buf->buf.start; + mmap_buf->buf.end = mmap_buf->buf.start + size; + mmap_buf->process = NULL; + + res = read(req->content_fd, mmap_buf->free_ptr, size); + if (res < 0) { + nxt_unit_req_alert(req, "failed to read content: %s (%d)", + strerror(errno), errno); + + nxt_unit_mmap_buf_free(mmap_buf); + + return NULL; + } + + if (res < (ssize_t) size) { + close(req->content_fd); + + req->content_fd = -1; + } + + nxt_unit_req_debug(req, "preread: read %d", (int) res); + + mmap_buf->buf.end = mmap_buf->buf.free + res; + + return mmap_buf; +} + + static ssize_t nxt_unit_buf_read(nxt_unit_buf_t **b, uint64_t *len, void *dst, size_t size) { u_char *p; size_t rest, copy, read; - nxt_unit_buf_t *buf; + nxt_unit_buf_t *buf, *last_buf; p = dst; rest = size; buf = *b; + last_buf = buf; while (buf != NULL) { + last_buf = buf; + copy = buf->end - buf->free; copy = nxt_min(rest, copy); @@ -2498,7 +2610,7 @@ nxt_unit_buf_read(nxt_unit_buf_t **b, uint64_t *len, void *dst, size_t size) buf = nxt_unit_buf_next(buf); } - *b = buf; + *b = last_buf; read = size - rest; diff --git a/src/nxt_unit.h b/src/nxt_unit.h index 8e9e3015..900f3ac2 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -103,6 +103,7 @@ struct nxt_unit_request_info_s { nxt_unit_buf_t *content_buf; uint64_t content_length; + int content_fd; void *data; }; diff --git a/test/unit/main.py b/test/unit/main.py index a7f5816a..69234dcc 100644 --- a/test/unit/main.py +++ b/test/unit/main.py @@ -219,6 +219,7 @@ class TestUnit(unittest.TestCase): '--pid', self.testdir + '/unit.pid', '--log', self.testdir + '/unit.log', '--control', 'unix:' + self.testdir + '/control.unit.sock', + '--tmp', self.testdir, ], stderr=log, ) -- cgit From f6d53f69b86b976c46af7a8554aa483d21f08e44 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Thu, 12 Mar 2020 18:13:07 +0300 Subject: Added version 1.16.0 CHANGES. --- CHANGES | 25 ++++++++++++++++++ docs/changes.xml | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/CHANGES b/CHANGES index e8e1350f..c44466e6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,29 @@ +Changes with Unit 1.16.0 12 Mar 2020 + + *) Feature: basic load-balancing support with round-robin. + + *) Feature: a "fallback" option that performs an alternative action if a + request can't be served from the "share" directory. + + *) Feature: reduced memory consumption by dumping large request bodies + to disk. + + *) Feature: stripping UTF-8 BOM and JavaScript-style comments from + uploaded JSON. + + *) Bugfix: negative address matching in router might work improperly in + combination with non-negative patterns. + + *) Bugfix: Java Spring applications failed to run; the bug had appeared + in 1.10.0. + + *) Bugfix: PHP 7.4 was broken if it was built with thread safety + enabled. + + *) Bugfix: compatibility issues with some Python applications. + + Changes with Unit 1.15.0 06 Feb 2020 *) Change: extensions of dynamically requested PHP scripts were diff --git a/docs/changes.xml b/docs/changes.xml index 3aa579fb..270b49d3 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -5,6 +5,84 @@ + + + + +NGINX Unit updated to 1.16.0. + + + + + + + + + + +basic load-balancing support with round-robin. + + + + + +a "fallback" option that performs an alternative action if a request can't be +served from the "share" directory. + + + + + +reduced memory consumption by dumping large request bodies to disk. + + + + + +stripping UTF-8 BOM and JavaScript-style comments from uploaded JSON. + + + + + +negative address matching in router might work improperly in combination with +non-negative patterns. + + + + + +Java Spring applications failed to run; the bug had appeared in 1.10.0. + + + + + +PHP 7.4 was broken if it was built with thread safety enabled. + + + + + +compatibility issues with some Python applications. + + + + + + " -ENV UNIT_VERSION 1.15.0-1~buster +ENV UNIT_VERSION 1.16.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.go1.11-dev b/pkg/docker/Dockerfile.go1.11-dev index 589680c8..ab9bb699 100644 --- a/pkg/docker/Dockerfile.go1.11-dev +++ b/pkg/docker/Dockerfile.go1.11-dev @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.15.0-1~buster +ENV UNIT_VERSION 1.16.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.minimal b/pkg/docker/Dockerfile.minimal index 3c3853eb..03fab2a2 100644 --- a/pkg/docker/Dockerfile.minimal +++ b/pkg/docker/Dockerfile.minimal @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.15.0-1~buster +ENV UNIT_VERSION 1.16.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.perl5.28 b/pkg/docker/Dockerfile.perl5.28 index 92a88516..f9b596f2 100644 --- a/pkg/docker/Dockerfile.perl5.28 +++ b/pkg/docker/Dockerfile.perl5.28 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.15.0-1~buster +ENV UNIT_VERSION 1.16.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.php7.3 b/pkg/docker/Dockerfile.php7.3 index 0180909b..e3c2bfbd 100644 --- a/pkg/docker/Dockerfile.php7.3 +++ b/pkg/docker/Dockerfile.php7.3 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.15.0-1~buster +ENV UNIT_VERSION 1.16.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.python2.7 b/pkg/docker/Dockerfile.python2.7 index 2942b0b5..065fc61b 100644 --- a/pkg/docker/Dockerfile.python2.7 +++ b/pkg/docker/Dockerfile.python2.7 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.15.0-1~buster +ENV UNIT_VERSION 1.16.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.python3.7 b/pkg/docker/Dockerfile.python3.7 index e8ca5c4a..d80d5533 100644 --- a/pkg/docker/Dockerfile.python3.7 +++ b/pkg/docker/Dockerfile.python3.7 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.15.0-1~buster +ENV UNIT_VERSION 1.16.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.ruby2.5 b/pkg/docker/Dockerfile.ruby2.5 index b706e542..3d141335 100644 --- a/pkg/docker/Dockerfile.ruby2.5 +++ b/pkg/docker/Dockerfile.ruby2.5 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.15.0-1~buster +ENV UNIT_VERSION 1.16.0-1~buster RUN set -x \ && apt-get update \ -- cgit From b3c8a7b33a29208e75dfe4f670cf81dac7b99ccc Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Thu, 12 Mar 2020 18:35:33 +0300 Subject: Added tag 1.16.0 for changeset 8bab088952dd --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 7891526f..ad451c6f 100644 --- a/.hgtags +++ b/.hgtags @@ -22,3 +22,4 @@ b391df5f0102aa6afe660cfc863729c1b1111c9e 1.12.0 3313bf222e6e0a91213946dfcbd70bb5079f4cef 1.13.0 6e28966ed1f26e119bf333229ea5e6686c60a469 1.14.0 801ac82f80fb2b2333f2c03ac9c3df6b7cec130a 1.15.0 +8bab088952dd9d7caa3d04fd4b3026cef26fcf7d 1.16.0 -- cgit