r1292, r1296 merge:

proxy_store and fastcgi_store, proxy_store_access and fastcgi_store_access

6 files changed, 403 insertions, 4 deletions

diff --git a/src/core/ngx_file.c b/src/core/ngx_file.c
index 92c74d699..fcb8ff080 100644
--- a/src/core/ngx_file.c
+++ b/src/core/ngx_file.c
@@ -293,6 +293,68 @@ ngx_conf_set_path_slot(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
 }
 
 
+char *
+ngx_conf_set_access_slot(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
+{
+    char  *confp = conf;
+
+    u_char      *p;
+    ngx_str_t   *value;
+    ngx_uint_t   i, right, shift, *access;
+
+    access = (ngx_uint_t *) (confp + cmd->offset);
+
+    if (*access != NGX_CONF_UNSET_UINT) {
+        return "is duplicate";
+    }
+
+    value = cf->args->elts;
+
+    *access = 0600;
+
+    for (i = 1; i < cf->args->nelts; i++) {
+
+        p = value[i].data;
+
+        if (ngx_strncmp(p, "user:", sizeof("user:") - 1) == 0) {
+            shift = 6;
+            p += sizeof("user:") - 1;
+
+        } else if (ngx_strncmp(p, "group:", sizeof("group:") - 1) == 0) {
+            shift = 3;
+            p += sizeof("group:") - 1;
+
+        } else if (ngx_strncmp(p, "all:", sizeof("all:") - 1) == 0) {
+            shift = 0;
+            p += sizeof("all:") - 1;
+
+        } else {
+            goto invalid;
+        }
+
+        if (ngx_strcmp(p, "rw") == 0) {
+            right = 6;
+
+        } else if (ngx_strcmp(p, "r") == 0) {
+            right = 4;
+
+        } else {
+            goto invalid;
+        }
+
+        *access |= right << shift;
+    }
+
+    return NGX_CONF_OK;
+
+invalid:
+
+    ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid value \"%V\"", &value[i]);
+
+    return NGX_CONF_ERROR;
+}
+
+
 ngx_int_t
 ngx_add_path(ngx_conf_t *cf, ngx_path_t **slot)
 {
diff --git a/src/core/ngx_file.h b/src/core/ngx_file.h
index e42489d8d..0f819a1b8 100644
--- a/src/core/ngx_file.h
+++ b/src/core/ngx_file.h
@@ -95,6 +95,7 @@ void ngx_init_temp_number(void);
 ngx_atomic_uint_t ngx_next_temp_number(ngx_uint_t collision);
 
 char *ngx_conf_set_path_slot(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
+char *ngx_conf_set_access_slot(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
 
 
 #define ngx_conf_merge_path_value(curr, prev, path, l1, l2, l3, clean, cf)    \
diff --git a/src/http/modules/ngx_http_fastcgi_module.c b/src/http/modules/ngx_http_fastcgi_module.c
index 246574b13..c1ce79ce7 100644
--- a/src/http/modules/ngx_http_fastcgi_module.c
+++ b/src/http/modules/ngx_http_fastcgi_module.c
@@ -124,6 +124,8 @@ static ngx_int_t ngx_http_fastcgi_script_name_variable(ngx_http_request_t *r,
 
 static char *ngx_http_fastcgi_pass(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
+static char *ngx_http_fastcgi_store(ngx_conf_t *cf, ngx_command_t *cmd,
+    void *conf);
 static char *ngx_http_fastcgi_lowat_check(ngx_conf_t *cf, void *post,
     void *data);
 
@@ -200,6 +202,20 @@ static ngx_command_t  ngx_http_fastcgi_commands[] = {
       offsetof(ngx_http_fastcgi_loc_conf_t, index),
       NULL },
 
+    { ngx_string("fastcgi_store"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+      ngx_http_fastcgi_store,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      0,
+      NULL },
+
+    { ngx_string("fastcgi_store_access"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE123,
+      ngx_conf_set_access_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_fastcgi_loc_conf_t, upstream.store_access),
+      NULL },
+
     { ngx_string("fastcgi_ignore_client_abort"),
       NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
       ngx_conf_set_flag_slot,
@@ -1628,11 +1644,15 @@ ngx_http_fastcgi_create_loc_conf(ngx_conf_t *cf)
      *     conf->upstream.schema = { 0, NULL };
      *     conf->upstream.uri = { 0, NULL };
      *     conf->upstream.location = NULL;
+     *     conf->upstream.store_lengths = NULL;
+     *     conf->upstream.store_values = NULL;
      *
      *     conf->index.len = 0;
      *     conf->index.data = NULL;
      */
 
+    conf->upstream.store = NGX_CONF_UNSET;
+    conf->upstream.store_access = NGX_CONF_UNSET_UINT;
     conf->upstream.buffering = NGX_CONF_UNSET;
     conf->upstream.ignore_client_abort = NGX_CONF_UNSET;
 
@@ -1677,6 +1697,19 @@ ngx_http_fastcgi_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child)
     ngx_http_script_compile_t     sc;
     ngx_http_script_copy_code_t  *copy;
 
+    if (conf->upstream.store != 0) {
+        ngx_conf_merge_value(conf->upstream.store,
+                                  prev->upstream.store, 0);
+
+        if (conf->upstream.store_lengths == NULL) {
+            conf->upstream.store_lengths = prev->upstream.store_lengths;
+            conf->upstream.store_values = prev->upstream.store_values;
+        }
+    }
+
+    ngx_conf_merge_uint_value(conf->upstream.store_access,
+                              prev->upstream.store_access, 0600);
+
     ngx_conf_merge_value(conf->upstream.buffering,
                               prev->upstream.buffering, 1);
 
@@ -2148,6 +2181,52 @@ ngx_http_fastcgi_pass(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
 
 
 static char *
+ngx_http_fastcgi_store(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
+{
+    ngx_http_fastcgi_loc_conf_t *flcf = conf;
+
+    ngx_str_t                  *value;
+    ngx_http_script_compile_t   sc;
+
+    if (flcf->upstream.store != NGX_CONF_UNSET || flcf->upstream.store_lengths)
+    {
+        return "is duplicate";
+    }
+
+    value = cf->args->elts;
+
+    if (ngx_strcmp(value[1].data, "on") == 0) {
+        flcf->upstream.store = 1;
+        return NGX_CONF_OK;
+    }
+
+    if (ngx_strcmp(value[1].data, "off") == 0) {
+        flcf->upstream.store = 0;
+        return NGX_CONF_OK;
+    }
+
+    /* include the terminating '\0' into script */
+    value[1].len++;
+
+    ngx_memzero(&sc, sizeof(ngx_http_script_compile_t));
+
+    sc.cf = cf;
+    sc.source = &value[1];
+    sc.lengths = &flcf->upstream.store_lengths;
+    sc.values = &flcf->upstream.store_values;
+    sc.variables = ngx_http_script_variables_count(&value[1]);;
+    sc.complete_lengths = 1;
+    sc.complete_values = 1;
+
+    if (ngx_http_script_compile(&sc) != NGX_OK) {
+        return NGX_CONF_ERROR;
+    }
+
+    return NGX_CONF_OK;
+}
+
+
+static char *
 ngx_http_fastcgi_lowat_check(ngx_conf_t *cf, void *post, void *data)
 {
 #if (NGX_FREEBSD)
diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c
index 3ea8006fe..c999adea2 100644
--- a/src/http/modules/ngx_http_proxy_module.c
+++ b/src/http/modules/ngx_http_proxy_module.c
@@ -105,6 +105,8 @@ static char *ngx_http_proxy_pass(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
 static char *ngx_http_proxy_redirect(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
+static char *ngx_http_proxy_store(ngx_conf_t *cf, ngx_command_t *cmd,
+    void *conf);
 
 static char *ngx_http_proxy_lowat_check(ngx_conf_t *cf, void *post, void *data);
 
@@ -154,6 +156,20 @@ static ngx_command_t  ngx_http_proxy_commands[] = {
       0,
       NULL },
 
+    { ngx_string("proxy_store"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+      ngx_http_proxy_store,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      0,
+      NULL },
+
+    { ngx_string("proxy_store_access"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE123,
+      ngx_conf_set_access_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_proxy_loc_conf_t, upstream.store_access),
+      NULL },
+
     { ngx_string("proxy_buffering"),
       NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
       ngx_conf_set_flag_slot,
@@ -1490,6 +1506,8 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf)
      *     conf->upstream.schema = { 0, NULL };
      *     conf->upstream.uri = { 0, NULL };
      *     conf->upstream.location = NULL;
+     *     conf->upstream.store_lengths = NULL;
+     *     conf->upstream.store_values = NULL;
      *
      *     conf->method = NULL;
      *     conf->headers_source = NULL;
@@ -1502,6 +1520,8 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf)
      *     conf->rewrite_locations = NULL;
      */
 
+    conf->upstream.store = NGX_CONF_UNSET;
+    conf->upstream.store_access = NGX_CONF_UNSET_UINT;
     conf->upstream.buffering = NGX_CONF_UNSET;
     conf->upstream.ignore_client_abort = NGX_CONF_UNSET;
 
@@ -1553,6 +1573,19 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child)
     ngx_http_script_compile_t     sc;
     ngx_http_script_copy_code_t  *copy;
 
+    if (conf->upstream.store != 0) {
+        ngx_conf_merge_value(conf->upstream.store,
+                                  prev->upstream.store, 0);
+
+        if (conf->upstream.store_lengths == NULL) {
+            conf->upstream.store_lengths = prev->upstream.store_lengths;
+            conf->upstream.store_values = prev->upstream.store_values;
+        }
+    }
+
+    ngx_conf_merge_uint_value(conf->upstream.store_access,
+                              prev->upstream.store_access, 0600);
+
     ngx_conf_merge_value(conf->upstream.buffering,
                               prev->upstream.buffering, 1);
 
@@ -2360,6 +2393,52 @@ ngx_http_proxy_redirect(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
 
 
 static char *
+ngx_http_proxy_store(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
+{
+    ngx_http_proxy_loc_conf_t *plcf = conf;
+
+    ngx_str_t                  *value;
+    ngx_http_script_compile_t   sc;
+
+    if (plcf->upstream.store != NGX_CONF_UNSET || plcf->upstream.store_lengths)
+    {
+        return "is duplicate";
+    }
+
+    value = cf->args->elts;
+
+    if (ngx_strcmp(value[1].data, "on") == 0) {
+        plcf->upstream.store = 1;
+        return NGX_CONF_OK;
+    }
+
+    if (ngx_strcmp(value[1].data, "off") == 0) {
+        plcf->upstream.store = 0;
+        return NGX_CONF_OK;
+    }
+
+    /* include the terminating '\0' into script */
+    value[1].len++;
+
+    ngx_memzero(&sc, sizeof(ngx_http_script_compile_t));
+
+    sc.cf = cf;
+    sc.source = &value[1];
+    sc.lengths = &plcf->upstream.store_lengths;
+    sc.values = &plcf->upstream.store_values;
+    sc.variables = ngx_http_script_variables_count(&value[1]);;
+    sc.complete_lengths = 1;
+    sc.complete_values = 1;
+
+    if (ngx_http_script_compile(&sc) != NGX_OK) {
+        return NGX_CONF_ERROR;
+    }
+
+    return NGX_CONF_OK;
+}
+
+
+static char *
 ngx_http_proxy_lowat_check(ngx_conf_t *cf, void *post, void *data)
 {
 #if (NGX_FREEBSD)
diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
index d01941a86..376b85fcd 100644
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -33,6 +33,8 @@ static ngx_int_t ngx_http_upstream_non_buffered_filter(void *data,
     ssize_t bytes);
 static void ngx_http_upstream_process_downstream(ngx_http_request_t *r);
 static void ngx_http_upstream_process_body(ngx_event_t *ev);
+static void ngx_http_upstream_store(ngx_http_request_t *r,
+    ngx_http_upstream_t *u);
 static void ngx_http_upstream_dummy_handler(ngx_event_t *wev);
 static void ngx_http_upstream_next(ngx_http_request_t *r,
     ngx_http_upstream_t *u, ngx_uint_t ft_type);
@@ -116,6 +118,12 @@ ngx_http_upstream_header_t  ngx_http_upstream_headers_in[] = {
                  ngx_http_upstream_copy_header_line,
                  offsetof(ngx_http_headers_out_t, date), 0 },
 
+    { ngx_string("Last-Modified"),
+                 ngx_http_upstream_process_header_line,
+                 offsetof(ngx_http_upstream_headers_in_t, last_modified),
+                 ngx_http_upstream_copy_header_line,
+                 offsetof(ngx_http_headers_out_t, last_modified), 0 },
+
     { ngx_string("Server"),
                  ngx_http_upstream_process_header_line,
                  offsetof(ngx_http_upstream_headers_in_t, server),
@@ -364,6 +372,8 @@ ngx_http_upstream_init(ngx_http_request_t *r)
     cln->data = r;
     u->cleanup = &cln->handler;
 
+    u->store = (u->conf->store || u->conf->store_lengths);
+
     ngx_http_upstream_connect(r, u);
 }
 
@@ -424,7 +434,7 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r,
             ev->error = 1;
         }
 
-        if (!u->cachable && u->peer.connection) {
+        if (!u->cachable && !u->store && u->peer.connection) {
             ngx_log_error(NGX_LOG_INFO, ev->log, ev->kq_errno,
                           "kevent() reported that client closed prematurely "
                           "connection, so upstream connection is closed too");
@@ -490,7 +500,7 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r,
     ev->eof = 1;
     c->error = 1;
 
-    if (!u->cachable && u->peer.connection) {
+    if (!u->cachable && !u->store && u->peer.connection) {
         ngx_log_error(NGX_LOG_INFO, ev->log, err,
                       "client closed prematurely connection, "
                       "so upstream connection is closed too");
@@ -1523,7 +1533,7 @@ ngx_http_upstream_send_response(ngx_http_request_t *r, ngx_http_upstream_t *u)
     p->pool = r->pool;
     p->log = c->log;
 
-    p->cachable = u->cachable;
+    p->cachable = u->cachable || u->store;
 
     p->temp_file = ngx_pcalloc(r->pool, sizeof(ngx_temp_file_t));
     if (p->temp_file == NULL) {
@@ -1536,8 +1546,9 @@ ngx_http_upstream_send_response(ngx_http_request_t *r, ngx_http_upstream_t *u)
     p->temp_file->path = u->conf->temp_path;
     p->temp_file->pool = r->pool;
 
-    if (u->cachable) {
+    if (u->cachable || u->store) {
         p->temp_file->persistent = 1;
+
     } else {
         p->temp_file->log_level = NGX_LOG_WARN;
         p->temp_file->warn = "an upstream response is buffered "
@@ -1552,6 +1563,7 @@ ngx_http_upstream_send_response(ngx_http_request_t *r, ngx_http_upstream_t *u)
         ngx_http_upstream_finalize_request(r, u, 0);
         return;
     }
+
     p->preread_bufs->buf = &u->buffer;
     p->preread_bufs->next = NULL;
     u->buffer.recycled = 1;
@@ -1559,11 +1571,13 @@ ngx_http_upstream_send_response(ngx_http_request_t *r, ngx_http_upstream_t *u)
     p->preread_size = u->buffer.last - u->buffer.pos;
 
     if (u->cachable) {
+
         p->buf_to_file = ngx_calloc_buf(r->pool);
         if (p->buf_to_file == NULL) {
             ngx_http_upstream_finalize_request(r, u, 0);
             return;
         }
+
         p->buf_to_file->pos = u->buffer.start;
         p->buf_to_file->last = u->buffer.pos;
         p->buf_to_file->temporary = 1;
@@ -1910,6 +1924,27 @@ ngx_http_upstream_process_body(ngx_event_t *ev)
 
     if (u->peer.connection) {
 
+        if (u->store) {
+
+            if (p->upstream_eof && u->headers_in.status_n == NGX_HTTP_OK) {
+
+                ngx_http_upstream_store(r, u);
+
+            } else if ((p->upstream_error
+                        || (p->upstream_eof
+                            && u->headers_in.status_n != NGX_HTTP_OK))
+                       && u->pipe->temp_file->file.fd != NGX_INVALID_FILE)
+            {
+                if (ngx_delete_file(u->pipe->temp_file->file.name.data)
+                    == NGX_FILE_ERROR)
+                {
+                    ngx_log_error(NGX_LOG_CRIT, r->connection->log, ngx_errno,
+                                  ngx_delete_file_n " \"%s\" failed",
+                                  u->pipe->temp_file->file.name.data);
+                }
+            }
+        }
+
 #if (NGX_HTTP_FILE_CACHE)
 
         if (p->upstream_done && u->cachable) {
@@ -1955,6 +1990,143 @@ ngx_http_upstream_process_body(ngx_event_t *ev)
 
 
 static void
+ngx_http_upstream_store(ngx_http_request_t *r, ngx_http_upstream_t *u)
+{
+    char             *failed;
+    u_char           *name;
+    size_t            root;
+    time_t            lm;
+    ngx_err_t         err;
+    ngx_str_t        *temp, path, *last_modified;
+    ngx_temp_file_t  *tf;
+
+    if (u->pipe->temp_file->file.fd == NGX_INVALID_FILE) {
+
+        /* create file for empty 200 response */
+
+        tf = ngx_pcalloc(r->pool, sizeof(ngx_temp_file_t));
+        if (tf == NULL) {
+            return;
+        }
+
+        tf->file.fd = NGX_INVALID_FILE;
+        tf->file.log = r->connection->log;
+        tf->path = u->conf->temp_path;
+        tf->pool = r->pool;
+        tf->persistent = 1;
+
+        if (ngx_create_temp_file(&tf->file, tf->path, tf->pool,
+                                 tf->persistent, tf->clean, tf->access)
+            != NGX_OK)
+        {
+            return;
+        }
+
+        u->pipe->temp_file = tf;
+    }
+
+    temp = &u->pipe->temp_file->file.name;
+
+#if !(NGX_WIN32)
+
+    if (ngx_change_file_access(temp->data, u->conf->store_access)
+        == NGX_FILE_ERROR)
+    {
+        err = ngx_errno;
+        failed = ngx_change_file_access_n;
+        name = temp->data;
+
+        goto failed;
+    }
+
+#endif
+
+    if (r->upstream->headers_in.last_modified) {
+
+        last_modified = &r->upstream->headers_in.last_modified->value;
+
+        lm = ngx_http_parse_time(last_modified->data, last_modified->len);
+
+        if (lm != NGX_ERROR) {
+            if (ngx_set_file_time(temp->data, u->pipe->temp_file->file.fd, lm)
+                != NGX_OK)
+            {
+                err = ngx_errno;
+                failed = ngx_set_file_time_n;
+                name = temp->data;
+
+                goto failed;
+            }
+        }
+    }
+
+    if (u->conf->store_lengths == NULL) {
+
+        ngx_http_map_uri_to_path(r, &path, &root, 0);
+
+    } else {
+        if (ngx_http_script_run(r, &path, u->conf->store_lengths->elts, 0,
+                                u->conf->store_values->elts)
+            == NULL)
+        {
+            return;
+        }
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "upstream stores \"%s\" to \"%s\"", temp->data, path.data);
+
+    failed = ngx_rename_file_n;
+    name = path.data;
+
+    if (ngx_rename_file(temp->data, path.data) != NGX_FILE_ERROR) {
+        return;
+    }
+
+    err = ngx_errno;
+
+    if (err == NGX_ENOENT) {
+
+        err = ngx_create_full_path(path.data,
+                                   ngx_dir_access(u->conf->store_access));
+        if (err == 0) {
+            if (ngx_rename_file(temp->data, path.data) != NGX_FILE_ERROR) {
+                return;
+            }
+
+            err = ngx_errno;
+        }
+    }
+
+#if (NGX_WIN32)
+
+    if (err == NGX_EEXIST) {
+        if (ngx_win32_rename_file(temp, &path, r->pool) != NGX_ERROR) {
+
+            if (ngx_rename_file(temp->data, path.data) != NGX_FILE_ERROR) {
+                return;
+            }
+        }
+
+        err = ngx_errno;
+    }
+
+#endif
+
+failed:
+
+    if (ngx_delete_file(temp->data) == NGX_FILE_ERROR) {
+        ngx_log_error(NGX_LOG_CRIT, r->connection->log, ngx_errno,
+                      ngx_delete_file_n " \"%s\" failed",
+                      temp->data);
+    }
+
+    ngx_log_error(NGX_LOG_CRIT, r->connection->log, err,
+                  "%s \"%s\" failed", failed, name);
+}
+
+
+static void
 ngx_http_upstream_dummy_handler(ngx_event_t *wev)
 {
     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, wev->log, 0,
diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h
index b2e3f3e93..b3bb1b4fd 100644
--- a/src/http/ngx_http_upstream.h
+++ b/src/http/ngx_http_upstream.h
@@ -118,6 +118,7 @@ typedef struct {
     size_t                          temp_file_write_size_conf;
 
     ngx_uint_t                      next_upstream;
+    ngx_uint_t                      store_access;
 
     ngx_bufs_t                      bufs;
 
@@ -140,6 +141,10 @@ typedef struct {
     ngx_str_t                       location;
     ngx_str_t                       url;  /* used in proxy_rewrite_location */
 
+    ngx_array_t                    *store_lengths;
+    ngx_array_t                    *store_values;
+
+    signed                          store:2;
     unsigned                        intercept_404:1;
     unsigned                        change_buffering:1;
 
@@ -237,6 +242,7 @@ struct ngx_http_upstream_s {
 
     ngx_http_cleanup_pt            *cleanup;
 
+    unsigned                        store:1;
     unsigned                        cachable:1;
     unsigned                        accel:1;