Klaus Demo nginx / 19298ec
introduce cache manager instead of cache cleaner Igor Sysoev 13 years ago
5 changed file(s) with 495 addition(s) and 215 deletion(s). Raw diff Collapse all Expand all
263263 }
264264
265265 path->len = 0;
266 path->cleaner = (ngx_path_cleaner_pt) cmd->post;
266 path->manager = (ngx_path_manager_pt) cmd->post;
267267 path->conf_file = cf->conf_file->file.name.data;
268268 path->line = cf->conf_file->line;
269269
323323 + init->level[1] + (init->level[1] ? 1 : 0)
324324 + init->level[2] + (init->level[2] ? 1 : 0);
325325
326 (*path)->cleaner = NULL;
326 (*path)->manager = NULL;
327327 (*path)->conf_file = NULL;
328328
329329 if (ngx_add_path(cf, path) != NGX_OK) {
2828 #define NGX_MAX_PATH_LEVEL 3
2929
3030
31 typedef time_t (*ngx_path_cleaner_pt) (void *data);
31 typedef time_t (*ngx_path_manager_pt) (void *data);
3232
3333
3434 typedef struct {
3636 size_t len;
3737 size_t level[3];
3838
39 ngx_path_cleaner_pt cleaner;
39 ngx_path_manager_pt manager;
4040 void *data;
4141
4242 u_char *conf_file;
4747 time_t expire;
4848 time_t valid_sec;
4949 size_t body_start;
50 off_t length;
5051 } ngx_http_file_cache_node_t;
5152
5253
99100
100101 ngx_path_t *path;
101102
103 ngx_atomic_t *cold;
104 off_t *size;
105
106 off_t max_size;
107 size_t bsize;
108
102109 time_t inactive;
103 time_t created;
104 time_t clean_time;
105 time_t next_clean_time;
110
111 ngx_msec_t last;
112 ngx_uint_t files;
106113
107114 ngx_shm_zone_t *shm_zone;
108115 };
99 #include <ngx_md5.h>
1010
1111
12 static ngx_int_t ngx_http_file_cache_exists(ngx_http_request_t *r,
13 ngx_http_file_cache_t *cache);
12 static ngx_int_t ngx_http_file_cache_exists(ngx_http_file_cache_t *cache,
13 ngx_http_cache_t *c);
1414 static ngx_http_file_cache_node_t *
1515 ngx_http_file_cache_lookup(ngx_http_file_cache_t *cache, u_char *key);
1616 static void ngx_http_file_cache_rbtree_insert_value(ngx_rbtree_node_t *temp,
1717 ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel);
1818 static void ngx_http_file_cache_cleanup(void *data);
19 static time_t ngx_http_file_cache_expire(ngx_http_file_cache_t *cache,
20 ngx_uint_t force);
21 static ngx_int_t ngx_http_file_cache_clean_noop(ngx_tree_ctx_t *ctx,
19 static time_t ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache);
20 static time_t ngx_http_file_cache_expire(ngx_http_file_cache_t *cache);
21 static void ngx_http_file_cache_delete(ngx_http_file_cache_t *cache,
22 ngx_queue_t *q, u_char *name);
23 static ngx_int_t
24 ngx_http_file_cache_manager_sleep(ngx_http_file_cache_t *cache);
25 static ngx_int_t ngx_http_file_cache_noop(ngx_tree_ctx_t *ctx,
2226 ngx_str_t *path);
23 static ngx_int_t ngx_http_file_cache_clean_file(ngx_tree_ctx_t *ctx,
27 static ngx_int_t ngx_http_file_cache_manage_file(ngx_tree_ctx_t *ctx,
28 ngx_str_t *path);
29 static ngx_int_t ngx_http_file_cache_add_file(ngx_tree_ctx_t *ctx,
30 ngx_str_t *path);
31 static ngx_int_t ngx_http_file_cache_add(ngx_http_file_cache_t *cache,
32 ngx_http_cache_t *c);
33 static ngx_int_t ngx_http_file_cache_delete_file(ngx_tree_ctx_t *ctx,
2434 ngx_str_t *path);
2535
2636
5262 cache->rbtree = ocache->rbtree;
5363 cache->queue = ocache->queue;
5464 cache->shpool = ocache->shpool;
55 cache->created = ocache->created;
65 cache->cold = ocache->cold;
66 cache->size = ocache->size;
67 cache->bsize = ocache->bsize;
68
69 cache->max_size /= cache->bsize;
5670
5771 return NGX_OK;
5872 }
7993
8094 ngx_queue_init(cache->queue);
8195
96 cache->cold = ngx_slab_alloc(cache->shpool, sizeof(ngx_atomic_t));
97 if (cache->cold == NULL) {
98 return NGX_ERROR;
99 }
100
101 *cache->cold = 1;
102
103 cache->size = ngx_slab_alloc(cache->shpool, sizeof(off_t));
104 if (cache->size == NULL) {
105 return NGX_ERROR;
106 }
107
108 *cache->size = 0;
109
110 cache->bsize = ngx_fs_bsize(cache->path->name.data);
111
112 cache->max_size /= cache->bsize;
113
82114 len = sizeof(" in cache keys zone \"\"") + shm_zone->name.len;
83115
84116 cache->shpool->log_ctx = ngx_slab_alloc(cache->shpool, len);
88120
89121 ngx_sprintf(cache->shpool->log_ctx, " in cache keys zone \"%V\"%Z",
90122 &shm_zone->name);
91
92 cache->created = ngx_time();
93123
94124 return NGX_OK;
95125 }
154184 return NGX_ERROR;
155185 }
156186
157 rc = ngx_http_file_cache_exists(r, cache);
187 rc = ngx_http_file_cache_exists(cache, c);
158188
159189 ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
160190 "http file cache exists: %i u:%ui e:%d",
171201 return rc;
172202 }
173203
174 now = ngx_time();
175
176 cold = (now - cache->created < cache->inactive) ? 1 : 0;
204 cold = *cache->cold;
177205
178206 if (rc == NGX_OK) {
179207
303331
304332 ngx_shmtx_lock(&cache->shpool->mutex);
305333
306 c->node->uses = c->min_uses;
307 c->node->body_start = c->body_start;
308 c->node->exists = 1;
334 if (!c->node->exists) {
335 c->node->uses = 1;
336 c->node->body_start = c->body_start;
337 c->node->exists = 1;
338
339 *cache->size += (c->length + cache->bsize - 1) / cache->bsize;
340 }
309341
310342 ngx_shmtx_unlock(&cache->shpool->mutex);
311343 }
344
345 now = ngx_time();
312346
313347 if (c->valid_sec < now) {
314348
327361
328362
329363 static ngx_int_t
330 ngx_http_file_cache_exists(ngx_http_request_t *r, ngx_http_file_cache_t *cache)
364 ngx_http_file_cache_exists(ngx_http_file_cache_t *cache, ngx_http_cache_t *c)
331365 {
332366 ngx_int_t rc;
333367 ngx_http_file_cache_node_t *fcn;
334368
335369 ngx_shmtx_lock(&cache->shpool->mutex);
336370
337 fcn = ngx_http_file_cache_lookup(cache, r->cache->key);
371 fcn = ngx_http_file_cache_lookup(cache, c->key);
338372
339373 if (fcn) {
340374 ngx_queue_remove(&fcn->queue);
355389
356390 if (fcn->exists) {
357391
358 r->cache->exists = fcn->exists;
359 r->cache->body_start = fcn->body_start;
392 c->exists = fcn->exists;
393 c->body_start = fcn->body_start;
360394
361395 rc = NGX_OK;
362396
363397 goto done;
364398 }
365399
366 if (fcn->uses >= r->cache->min_uses) {
367
368 r->cache->exists = fcn->exists;
369 r->cache->body_start = fcn->body_start;
400 if (fcn->uses >= c->min_uses) {
401
402 c->exists = fcn->exists;
403 c->body_start = fcn->body_start;
370404
371405 rc = NGX_OK;
372406
382416 if (fcn == NULL) {
383417 ngx_shmtx_unlock(&cache->shpool->mutex);
384418
385 (void) ngx_http_file_cache_expire(cache, 1);
419 ngx_http_file_cache_forced_expire(cache);
386420
387421 ngx_shmtx_lock(&cache->shpool->mutex);
388422
394428 }
395429 }
396430
397 ngx_memcpy((u_char *) &fcn->node.key, r->cache->key,
398 sizeof(ngx_rbtree_key_t));
399
400 ngx_memcpy(fcn->key, &r->cache->key[sizeof(ngx_rbtree_key_t)],
431 ngx_memcpy((u_char *) &fcn->node.key, c->key, sizeof(ngx_rbtree_key_t));
432
433 ngx_memcpy(fcn->key, &c->key[sizeof(ngx_rbtree_key_t)],
401434 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t));
402435
403436 ngx_rbtree_insert(cache->rbtree, &fcn->node);
421454
422455 ngx_queue_insert_head(cache->queue, &fcn->queue);
423456
424 r->cache->uniq = fcn->uniq;
425 r->cache->uses = fcn->uses;
426 r->cache->error = fcn->error;
427 r->cache->node = fcn;
457 c->uniq = fcn->uniq;
458 c->uses = fcn->uses;
459 c->error = fcn->error;
460 c->node = fcn;
428461
429462 failed:
430463
566599 void
567600 ngx_http_file_cache_update(ngx_http_request_t *r, ngx_temp_file_t *tf)
568601 {
602 off_t size;
569603 ngx_int_t rc;
570604 ngx_file_uniq_t uniq;
571605 ngx_file_info_t fi;
615649 }
616650 }
617651
652 size = (c->length + cache->bsize - 1) / cache->bsize;
653
618654 ngx_shmtx_lock(&cache->shpool->mutex);
619655
620656 c->node->count--;
621657 c->node->uniq = uniq;
622658 c->node->body_start = c->body_start;
659
660 size = size - (c->node->length + cache->bsize - 1) / cache->bsize;
661
662 c->node->length = c->length;
663
664 *cache->size += size;
623665
624666 if (rc == NGX_OK) {
625667 c->node->exists = 1;
756798
757799
758800 static time_t
759 ngx_http_file_cache_expire(ngx_http_file_cache_t *cache, ngx_uint_t forced)
760 {
761 u_char *name, *p;
801 ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache)
802 {
803 u_char *name;
762804 size_t len;
763 time_t now, wait;
805 time_t wait;
764806 ngx_uint_t tries;
765807 ngx_path_t *path;
766808 ngx_queue_t *q;
767809 ngx_http_file_cache_node_t *fcn;
810
811 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
812 "http file cache forced expire");
813
814 path = cache->path;
815 len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN;
816
817 name = ngx_alloc(len + 1, ngx_cycle->log);
818 if (name == NULL) {
819 return 60;
820 }
821
822 ngx_memcpy(name, path->name.data, path->name.len);
823
824 wait = 60;
825 tries = 0;
826
827 ngx_shmtx_lock(&cache->shpool->mutex);
828
829 for (q = ngx_queue_last(cache->queue);
830 q != ngx_queue_sentinel(cache->queue);
831 q = ngx_queue_prev(q))
832 {
833 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue);
834
835 ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
836 "http file cache forced expire: #%d %d %02xd%02xd%02xd%02xd",
837 fcn->count, fcn->exists,
838 fcn->key[0], fcn->key[1], fcn->key[2], fcn->key[3]);
839
840 if (fcn->count) {
841
842 if (tries++ < 20) {
843 continue;
844 }
845
846 wait = 1;
847
848 break;
849 }
850
851 if (!fcn->exists) {
852
853 ngx_queue_remove(q);
854 ngx_rbtree_delete(cache->rbtree, &fcn->node);
855 ngx_slab_free_locked(cache->shpool, fcn);
856
857 break;
858 }
859
860 ngx_http_file_cache_delete(cache, q, name);
861
862 break;
863 }
864
865 ngx_shmtx_unlock(&cache->shpool->mutex);
866
867 ngx_free(name);
868
869 return wait;
870 }
871
872
873 static time_t
874 ngx_http_file_cache_expire(ngx_http_file_cache_t *cache)
875 {
876 u_char *name, *p;
877 size_t len;
878 time_t now, wait;
879 ngx_path_t *path;
880 ngx_queue_t *q;
881 ngx_http_file_cache_node_t *fcn;
768882 u_char key[2 * NGX_HTTP_CACHE_KEY_LEN];
769883
770884 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
773887 path = cache->path;
774888 len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN;
775889
890 name = ngx_alloc(len + 1, ngx_cycle->log);
891 if (name == NULL) {
892 return 60;
893 }
894
895 ngx_memcpy(name, path->name.data, path->name.len);
896
776897 now = ngx_time();
777898
778899 ngx_shmtx_lock(&cache->shpool->mutex);
779900
780 tries = 0;
781
782901 for ( ;; ) {
783902
784903 if (ngx_queue_empty(cache->queue)) {
785 wait = cache->inactive;
904 wait = 60;
786905 break;
787906 }
788907
790909
791910 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue);
792911
793 if (!forced) {
794 wait = fcn->expire - now;
795
796 if (wait > 0) {
797 break;
798 }
912 wait = fcn->expire - now;
913
914 if (wait > 0) {
915 wait = wait > 60 ? 60 : wait;
916 break;
799917 }
800918
801919 ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
805923
806924 if (fcn->count) {
807925
808 if (!forced) {
809
810 p = ngx_hex_dump(key, (u_char *) &fcn->node.key,
811 sizeof(ngx_rbtree_key_t));
812 (void) ngx_hex_dump(p, fcn->key,
813 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t));
814
815 /*
816 * abnormally exited workers may leave locked cache entries,
817 * and although it may be safe to remove them completely,
818 * we prefer to remove them from inactive queue and rbtree
819 * only, and to allow other leaks
820 */
821
822 ngx_queue_remove(q);
823
824 ngx_rbtree_delete(cache->rbtree, &fcn->node);
825
826 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
926 p = ngx_hex_dump(key, (u_char *) &fcn->node.key,
927 sizeof(ngx_rbtree_key_t));
928
929 len = NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t);
930 (void) ngx_hex_dump(p, fcn->key, len);
931
932 /*
933 * abnormally exited workers may leave locked cache entries,
934 * and although it may be safe to remove them completely,
935 * we prefer to remove them from inactive queue and rbtree
936 * only, and to allow other leaks
937 */
938
939 ngx_queue_remove(q);
940 ngx_rbtree_delete(cache->rbtree, &fcn->node);
941
942 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
827943 "ignore long locked inactive cache entry %*s, count:%d",
828944 2 * NGX_HTTP_CACHE_KEY_LEN, key, fcn->count);
829 }
830
831 if (tries++ < 10) {
832 continue;
833 }
834
835 wait = 1;
836 break;
837 }
838
839 forced = 0;
945
946 continue;
947 }
840948
841949 if (!fcn->exists) {
842950
843951 ngx_queue_remove(q);
844
845952 ngx_rbtree_delete(cache->rbtree, &fcn->node);
846
847953 ngx_slab_free_locked(cache->shpool, fcn);
848954
849955 continue;
850956 }
851957
852 name = ngx_alloc(len + 1, ngx_cycle->log);
853 if (name == NULL) {
854 wait = 60;
855 break;
856 }
857
858 ngx_memcpy(name, path->name.data, path->name.len);
859
860 p = name + path->name.len + 1 + path->len;
861 p = ngx_hex_dump(p, (u_char *) &fcn->node.key,
862 sizeof(ngx_rbtree_key_t));
863 p = ngx_hex_dump(p, fcn->key,
864 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t));
865 *p = '\0';
866
867 ngx_queue_remove(q);
868
869 ngx_rbtree_delete(cache->rbtree, &fcn->node);
870
871 ngx_slab_free_locked(cache->shpool, fcn);
872
873 ngx_shmtx_unlock(&cache->shpool->mutex);
874
875 ngx_create_hashed_filename(path, name, len);
876
877 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
878 "http file cache expire: \"%s\"", name);
879
880 if (ngx_delete_file(name) == NGX_FILE_ERROR) {
881 ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno,
882 ngx_delete_file_n " \"%s\" failed", name);
883 }
884
885 ngx_free(name);
886
887 ngx_shmtx_lock(&cache->shpool->mutex);
958 ngx_http_file_cache_delete(cache, q, name);
888959 }
889960
890961 ngx_shmtx_unlock(&cache->shpool->mutex);
891962
963 ngx_free(name);
964
892965 return wait;
893966 }
894967
895968
896 time_t
897 ngx_http_file_cache_cleaner(void *data)
969 static void
970 ngx_http_file_cache_delete(ngx_http_file_cache_t *cache, ngx_queue_t *q,
971 u_char *name)
972 {
973 u_char *p;
974 size_t len;
975 ngx_path_t *path;
976 ngx_http_file_cache_node_t *fcn;
977
978 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue);
979
980 *cache->size -= (fcn->length + cache->bsize - 1) / cache->bsize;
981
982 path = cache->path;
983
984 p = name + path->name.len + 1 + path->len;
985
986 p = ngx_hex_dump(p, (u_char *) &fcn->node.key, sizeof(ngx_rbtree_key_t));
987
988 len = NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t);
989 p = ngx_hex_dump(p, fcn->key, len);
990 *p = '\0';
991
992 ngx_queue_remove(q);
993
994 ngx_rbtree_delete(cache->rbtree, &fcn->node);
995
996 ngx_slab_free_locked(cache->shpool, fcn);
997
998 ngx_shmtx_unlock(&cache->shpool->mutex);
999
1000 len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN;
1001
1002 ngx_create_hashed_filename(path, name, len);
1003
1004 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
1005 "http file cache expire: \"%s\"", name);
1006
1007 if (ngx_delete_file(name) == NGX_FILE_ERROR) {
1008 ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno,
1009 ngx_delete_file_n " \"%s\" failed", name);
1010 }
1011
1012 ngx_shmtx_lock(&cache->shpool->mutex);
1013 }
1014
1015
1016 static time_t
1017 ngx_http_file_cache_manager(void *data)
8981018 {
8991019 ngx_http_file_cache_t *cache = data;
9001020
901 time_t now, next;
1021 off_t size;
1022 time_t next;
9021023 ngx_tree_ctx_t tree;
9031024
904 now = ngx_time();
905
906 if (now >= cache->next_clean_time
907 && now >= cache->created + cache->inactive)
908 {
909 ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0,
910 "clean unused cache files");
1025 if (*cache->cold) {
1026
1027 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
1028 "http file cache manager update");
9111029
9121030 tree.init_handler = NULL;
913 tree.file_handler = ngx_http_file_cache_clean_file;
914 tree.pre_tree_handler = ngx_http_file_cache_clean_noop;
915 tree.post_tree_handler = ngx_http_file_cache_clean_noop;
916 tree.spec_handler = ngx_http_file_cache_clean_file;
1031 tree.file_handler = ngx_http_file_cache_manage_file;
1032 tree.pre_tree_handler = ngx_http_file_cache_noop;
1033 tree.post_tree_handler = ngx_http_file_cache_noop;
1034 tree.spec_handler = ngx_http_file_cache_delete_file;
9171035 tree.data = cache;
9181036 tree.alloc = 0;
9191037 tree.log = ngx_cycle->log;
9201038
921 (void) ngx_walk_tree(&tree, &cache->path->name);
1039 cache->last = ngx_current_msec;
1040 cache->files = 0;
1041
1042 if (ngx_walk_tree(&tree, &cache->path->name) == NGX_ABORT) {
1043 return 60;
1044 }
1045
1046 *cache->cold = 0;
1047
1048 ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0,
1049 "http file cache: %V %.3fM, bsize: %uz",
1050 &cache->path->name,
1051 ((double) *cache->size * cache->bsize) / (1024 * 1024),
1052 cache->bsize);
1053 }
1054
1055 next = ngx_http_file_cache_expire(cache);
1056
1057 cache->last = ngx_current_msec;
1058 cache->files = 0;
1059
1060 for ( ;; ) {
1061 ngx_shmtx_lock(&cache->shpool->mutex);
1062
1063 size = *cache->size;
1064
1065 ngx_shmtx_unlock(&cache->shpool->mutex);
1066
1067 if (size < cache->max_size) {
1068 return next;
1069 }
1070
1071 next = ngx_http_file_cache_forced_expire(cache);
1072
1073 if (ngx_http_file_cache_manager_sleep(cache) != NGX_OK) {
1074 return next;
1075 }
1076 }
1077 }
1078
1079
1080 static ngx_int_t
1081 ngx_http_file_cache_manager_sleep(ngx_http_file_cache_t *cache)
1082 {
1083 ngx_msec_t elapsed;
1084
1085 if (cache->files++ > 100) {
9221086
9231087 ngx_time_update(0, 0);
9241088
925 next = ngx_next_time(cache->clean_time);
926
927 if (next == -1) {
928 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, ngx_errno,
929 ngx_next_time_n " failed");
930 return 60;
931 }
932
933 cache->next_clean_time = next;
934 }
935
936 next = ngx_http_file_cache_expire(cache, 0);
937
938 now = ngx_time();
939
940 return (now + next < cache->next_clean_time) ? next:
941 cache->next_clean_time - now;
1089 elapsed = ngx_abs((ngx_msec_int_t) (ngx_current_msec - cache->last));
1090
1091 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
1092 "http file cache manager time: %M", elapsed);
1093
1094 if (elapsed > 200) {
1095
1096 /*
1097 * if processing 100 files takes more than 200ms,
1098 * it seems that many operations require disk i/o,
1099 * therefore sleep 200ms
1100 */
1101
1102 ngx_msleep(200);
1103
1104 ngx_time_update(0, 0);
1105 }
1106
1107 cache->last = ngx_current_msec;
1108 cache->files = 0;
1109 }
1110
1111 return (ngx_quit || ngx_terminate) ? NGX_ABORT : NGX_OK;
9421112 }
9431113
9441114
9451115 static ngx_int_t
946 ngx_http_file_cache_clean_noop(ngx_tree_ctx_t *ctx, ngx_str_t *path)
1116 ngx_http_file_cache_noop(ngx_tree_ctx_t *ctx, ngx_str_t *path)
9471117 {
9481118 return NGX_OK;
9491119 }
9501120
9511121
9521122 static ngx_int_t
953 ngx_http_file_cache_clean_file(ngx_tree_ctx_t *ctx, ngx_str_t *path)
954 {
955 u_char *p, key[2 * NGX_HTTP_CACHE_KEY_LEN];
956 ngx_int_t n;
957 ngx_uint_t i;
958 ngx_http_file_cache_t *cache;
959 ngx_http_file_cache_node_t *node;
960
961 if (path->len < 2 * NGX_HTTP_CACHE_KEY_LEN) {
962 goto clean;
963 }
964
965 p = &path->data[path->len - 2 * NGX_HTTP_CACHE_KEY_LEN];
1123 ngx_http_file_cache_manage_file(ngx_tree_ctx_t *ctx, ngx_str_t *path)
1124 {
1125 ngx_http_file_cache_t *cache;
1126
1127 cache = ctx->data;
1128
1129 if (ngx_http_file_cache_add_file(ctx, path) != NGX_OK) {
1130 (void) ngx_http_file_cache_delete_file(ctx, path);
1131 }
1132
1133 return ngx_http_file_cache_manager_sleep(cache);
1134 }
1135
1136
1137 static ngx_int_t
1138 ngx_http_file_cache_add_file(ngx_tree_ctx_t *ctx, ngx_str_t *name)
1139 {
1140 u_char *p;
1141 ngx_fd_t fd;
1142 ngx_int_t n;
1143 ngx_uint_t i;
1144 ngx_file_info_t fi;
1145 ngx_http_cache_t c;
1146 ngx_http_file_cache_t *cache;
1147 ngx_http_file_cache_header_t h;
1148
1149 if (name->len < 2 * NGX_HTTP_CACHE_KEY_LEN) {
1150 return NGX_ERROR;
1151 }
1152
1153 ngx_memzero(&c, sizeof(ngx_http_cache_t));
1154
1155 fd = ngx_open_file(name->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
1156
1157 if (fd == NGX_INVALID_FILE) {
1158 ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno,
1159 ngx_open_file_n " \"%s\" failed", name->data);
1160 return NGX_ERROR;
1161 }
1162
1163 c.file.fd = fd;
1164 c.file.name = *name;
1165 c.file.log = ctx->log;
1166
1167 n = ngx_read_file(&c.file, (u_char *) &h,
1168 sizeof(ngx_http_file_cache_header_t), 0);
1169 if (n == NGX_ERROR) {
1170 return NGX_ERROR;
1171 }
1172
1173 if ((size_t) n < sizeof(ngx_http_file_cache_header_t)) {
1174 ngx_log_error(NGX_LOG_CRIT, ctx->log, 0,
1175 "cache file \"%s\" is too small", name->data);
1176 return NGX_ERROR;
1177 }
1178
1179 if (ngx_fd_info(fd, &fi) == NGX_FILE_ERROR) {
1180 ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno,
1181 ngx_fd_info_n " \"%s\" failed", name->data);
1182
1183 } else {
1184 c.uniq = ngx_file_uniq(&fi);
1185 c.valid_sec = h.valid_sec;
1186 c.valid_msec = h.valid_msec;
1187 c.body_start = h.body_start;
1188 c.length = ngx_file_size(&fi);
1189 }
1190
1191 if (ngx_close_file(fd) == NGX_FILE_ERROR) {
1192 ngx_log_error(NGX_LOG_ALERT, ctx->log, ngx_errno,
1193 ngx_close_file_n " \"%s\" failed", name->data);
1194 }
1195
1196 if (c.body_start == 0) {
1197 return NGX_ERROR;
1198 }
1199
1200 p = &name->data[name->len - 2 * NGX_HTTP_CACHE_KEY_LEN];
9661201
9671202 for (i = 0; i < NGX_HTTP_CACHE_KEY_LEN; i++) {
9681203 n = ngx_hextoi(p, 2);
9691204
9701205 if (n == NGX_ERROR) {
971 goto clean;
1206 return NGX_ERROR;
9721207 }
9731208
9741209 p += 2;
9751210
976 key[i] = (u_char) n;
1211 c.key[i] = (u_char) n;
9771212 }
9781213
9791214 cache = ctx->data;
9801215
1216 return ngx_http_file_cache_add(cache, &c);
1217 }
1218
1219
1220 static ngx_int_t
1221 ngx_http_file_cache_add(ngx_http_file_cache_t *cache, ngx_http_cache_t *c)
1222 {
1223 ngx_http_file_cache_node_t *fcn;
1224
9811225 ngx_shmtx_lock(&cache->shpool->mutex);
9821226
983 node = ngx_http_file_cache_lookup(cache, key);
1227 fcn = ngx_http_file_cache_lookup(cache, c->key);
1228
1229 if (fcn == NULL) {
1230
1231 fcn = ngx_slab_alloc_locked(cache->shpool,
1232 sizeof(ngx_http_file_cache_node_t));
1233 if (fcn == NULL) {
1234 ngx_shmtx_unlock(&cache->shpool->mutex);
1235 return NGX_ERROR;
1236 }
1237
1238 ngx_memcpy((u_char *) &fcn->node.key, c->key, sizeof(ngx_rbtree_key_t));
1239
1240 ngx_memcpy(fcn->key, &c->key[sizeof(ngx_rbtree_key_t)],
1241 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t));
1242
1243 ngx_rbtree_insert(cache->rbtree, &fcn->node);
1244
1245 fcn->uses = 1;
1246 fcn->count = 0;
1247 fcn->valid_msec = c->valid_msec;
1248 fcn->error = 0;
1249 fcn->exists = 1;
1250 fcn->uniq = c->uniq;
1251 fcn->valid_sec = c->valid_sec;
1252 fcn->body_start = c->body_start;
1253 fcn->length = c->length;
1254
1255 *cache->size += (c->length + cache->bsize - 1) / cache->bsize;
1256
1257 } else {
1258 ngx_queue_remove(&fcn->queue);
1259 }
1260
1261 fcn->expire = ngx_time() + cache->inactive;
1262
1263 ngx_queue_insert_head(cache->queue, &fcn->queue);
9841264
9851265 ngx_shmtx_unlock(&cache->shpool->mutex);
9861266
987 if (node) {
988 return NGX_OK;
989 }
990
991 clean:
992
1267 return NGX_OK;
1268 }
1269
1270
1271 static ngx_int_t
1272 ngx_http_file_cache_delete_file(ngx_tree_ctx_t *ctx, ngx_str_t *path)
1273 {
9931274 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ctx->log, 0,
994 "http file cache clean: \"%s\"", path->data);
1275 "http file cache delete: \"%s\"", path->data);
9951276
9961277 if (ngx_delete_file(path->data) == NGX_FILE_ERROR) {
9971278 ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno,
9981279 ngx_delete_file_n " \"%s\" failed", path->data);
999 return NGX_ERROR;
10001280 }
10011281
10021282 return NGX_OK;
10281308 char *
10291309 ngx_http_file_cache_set_slot(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
10301310 {
1311 off_t max_size;
10311312 u_char *last, *p;
1032 time_t inactive, clean_time, next;
1313 time_t inactive;
10331314 ssize_t size;
10341315 ngx_str_t s, name, *value;
10351316 ngx_uint_t i, n;
10461327 }
10471328
10481329 inactive = 600;
1049 clean_time = 5 * 60 * 60;
10501330
10511331 name.len = 0;
10521332 size = 0;
1333 max_size = NGX_MAX_OFF_T_VALUE;
10531334
10541335 value = cf->args->elts;
10551336
11521433 continue;
11531434 }
11541435
1155 if (ngx_strncmp(value[i].data, "clean_time=", 11) == 0) {
1156
1157 s.len = value[i].len - 11;
1158 s.data = value[i].data + 11;
1159
1160 clean_time = ngx_parse_time(&s, 1);
1161 if (clean_time < 0 || clean_time > 24 * 60 * 60) {
1436 if (ngx_strncmp(value[i].data, "max_size=", 9) == 0) {
1437
1438 s.len = value[i].len - 9;
1439 s.data = value[i].data + 9;
1440
1441 max_size = ngx_parse_offset(&s);
1442 if (max_size < 0) {
11621443 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
1163 "invalid clean_time value \"%V\"", &value[i]);
1444 "invalid max_size value \"%V\"", &value[i]);
11641445 return NGX_CONF_ERROR;
11651446 }
11661447
11791460 return NGX_CONF_ERROR;
11801461 }
11811462
1182 cache->path->cleaner = ngx_http_file_cache_cleaner;
1463 cache->path->manager = ngx_http_file_cache_manager;
11831464 cache->path->data = cache;
11841465
11851466 if (ngx_add_path(cf, &cache->path) != NGX_OK) {
11971478 return NGX_CONF_ERROR;
11981479 }
11991480
1200 next = ngx_next_time(clean_time);
1201
1202 if (next == -1) {
1203 ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno,
1204 ngx_next_time_n " failed");
1205 return NGX_CONF_ERROR;
1206 }
12071481
12081482 cache->shm_zone->init = ngx_http_file_cache_init;
12091483 cache->shm_zone->data = cache;
12101484
12111485 cache->inactive = inactive;
1212 cache->clean_time = clean_time;
1213 cache->next_clean_time = next;
1486 cache->max_size = max_size;
12141487
12151488 return NGX_CONF_OK;
12161489 }
1111
1212 static void ngx_start_worker_processes(ngx_cycle_t *cycle, ngx_int_t n,
1313 ngx_int_t type);
14 static void ngx_start_cleaner_process(ngx_cycle_t *cycle, ngx_int_t type);
14 static void ngx_start_cache_manager_process(ngx_cycle_t *cycle, ngx_int_t type);
1515 static void ngx_signal_worker_processes(ngx_cycle_t *cycle, int signo);
1616 static ngx_uint_t ngx_reap_children(ngx_cycle_t *cycle);
1717 static void ngx_master_process_exit(ngx_cycle_t *cycle);
2323 static void ngx_wakeup_worker_threads(ngx_cycle_t *cycle);
2424 static ngx_thread_value_t ngx_worker_thread_cycle(void *data);
2525 #endif
26 static void ngx_cleaner_process_cycle(ngx_cycle_t *cycle, void *data);
27 static void ngx_cleaner_process_handler(ngx_event_t *ev);
26 static void ngx_cache_manager_process_cycle(ngx_cycle_t *cycle, void *data);
27 static void ngx_cache_manager_process_handler(ngx_event_t *ev);
2828
2929
3030 ngx_uint_t ngx_process;
121121
122122 ngx_start_worker_processes(cycle, ccf->worker_processes,
123123 NGX_PROCESS_RESPAWN);
124 ngx_start_cleaner_process(cycle, NGX_PROCESS_RESPAWN);
124 ngx_start_cache_manager_process(cycle, NGX_PROCESS_RESPAWN);
125125
126126 ngx_new_binary = 0;
127127 delay = 0;
202202 if (ngx_new_binary) {
203203 ngx_start_worker_processes(cycle, ccf->worker_processes,
204204 NGX_PROCESS_RESPAWN);
205 ngx_start_cleaner_process(cycle, NGX_PROCESS_RESPAWN);
205 ngx_start_cache_manager_process(cycle, NGX_PROCESS_RESPAWN);
206206 ngx_noaccepting = 0;
207207
208208 continue;
221221 ngx_core_module);
222222 ngx_start_worker_processes(cycle, ccf->worker_processes,
223223 NGX_PROCESS_JUST_RESPAWN);
224 ngx_start_cleaner_process(cycle, NGX_PROCESS_JUST_RESPAWN);
224 ngx_start_cache_manager_process(cycle, NGX_PROCESS_JUST_RESPAWN);
225225 live = 1;
226226 ngx_signal_worker_processes(cycle,
227227 ngx_signal_value(NGX_SHUTDOWN_SIGNAL));
231231 ngx_restart = 0;
232232 ngx_start_worker_processes(cycle, ccf->worker_processes,
233233 NGX_PROCESS_RESPAWN);
234 ngx_start_cleaner_process(cycle, NGX_PROCESS_RESPAWN);
234 ngx_start_cache_manager_process(cycle, NGX_PROCESS_RESPAWN);
235235 live = 1;
236236 }
237237
359359
360360
361361 static void
362 ngx_start_cleaner_process(ngx_cycle_t *cycle, ngx_int_t type)
362 ngx_start_cache_manager_process(ngx_cycle_t *cycle, ngx_int_t type)
363363 {
364364 ngx_int_t i;
365365 ngx_uint_t n;
368368
369369 path = ngx_cycle->pathes.elts;
370370 for (n = 0; n < ngx_cycle->pathes.nelts; n++) {
371 if (path[n]->cleaner) {
371 if (path[n]->manager) {
372372 goto start;
373373 }
374374 }
379379
380380 ch.command = NGX_CMD_OPEN_CHANNEL;
381381
382 ngx_spawn_process(cycle, ngx_cleaner_process_cycle, NULL,
383 "cleaner process", type);
382 ngx_spawn_process(cycle, ngx_cache_manager_process_cycle, NULL,
383 "cache manager process", type);
384384
385385 ch.pid = ngx_processes[ngx_process_slot].pid;
386386 ch.slot = ngx_process_slot;
12621262
12631263
12641264 static void
1265 ngx_cleaner_process_cycle(ngx_cycle_t *cycle, void *data)
1265 ngx_cache_manager_process_cycle(ngx_cycle_t *cycle, void *data)
12661266 {
12671267 void *ident[4];
12681268 ngx_event_t ev;
12741274 ngx_close_listening_sockets(cycle);
12751275
12761276 ngx_memzero(&ev, sizeof(ngx_event_t));
1277 ev.handler = ngx_cleaner_process_handler;
1277 ev.handler = ngx_cache_manager_process_handler;
12781278 ev.data = ident;
12791279 ev.log = cycle->log;
12801280 ident[3] = (void *) -1;
12811281
12821282 ngx_use_accept_mutex = 0;
12831283
1284 ngx_setproctitle("cleaner process");
1284 ngx_setproctitle("cache manager process");
12851285
12861286 ngx_add_timer(&ev, 0);
12871287
13041304
13051305
13061306 static void
1307 ngx_cleaner_process_handler(ngx_event_t *ev)
1307 ngx_cache_manager_process_handler(ngx_event_t *ev)
13081308 {
13091309 time_t next, n;
13101310 ngx_uint_t i;
13151315 path = ngx_cycle->pathes.elts;
13161316 for (i = 0; i < ngx_cycle->pathes.nelts; i++) {
13171317
1318 if (path[i]->cleaner) {
1319 n = path[i]->cleaner(path[i]->data);
1318 if (path[i]->manager) {
1319 n = path[i]->manager(path[i]->data);
13201320
13211321 next = (n <= next) ? n : next;
13221322