Klaus Demo nginx / 67cd336
slab allocator in shared memory Igor Sysoev 15 years ago
21 changed file(s) with 893 addition(s) and 133 deletion(s). Raw diff Collapse all Expand all
2222 src/core/ngx_crc32.h \
2323 src/core/ngx_rbtree.h \
2424 src/core/ngx_radix_tree.h \
25 src/core/ngx_slab.h \
2526 src/core/ngx_times.h \
2627 src/core/ngx_shmtx.h \
2728 src/core/ngx_connection.h \
4546 src/core/ngx_crc32.c \
4647 src/core/ngx_rbtree.c \
4748 src/core/ngx_radix_tree.c \
49 src/core/ngx_slab.c \
4850 src/core/ngx_times.c \
4951 src/core/ngx_shmtx.c \
5052 src/core/ngx_connection.c \
620620 ngx_core_conf_t *ccf = conf;
621621
622622 #if !(NGX_WIN32)
623 ngx_str_t lock_file;
623624 struct passwd *pwd;
624625 struct group *grp;
625626 #endif
696697 ngx_memcpy(ngx_cpymem(ccf->oldpid.data, ccf->pid.data, ccf->pid.len),
697698 NGX_OLDPID_EXT, sizeof(NGX_OLDPID_EXT));
698699
700
699701 if (ccf->lock_file.len == 0) {
700702 ccf->lock_file.len = sizeof(NGX_LOCK_PATH) - 1;
701703 ccf->lock_file.data = (u_char *) NGX_LOCK_PATH;
703705
704706 if (ngx_conf_full_name(cycle, &ccf->lock_file) == NGX_ERROR) {
705707 return NGX_CONF_ERROR;
708 }
709
710 lock_file = cycle->old_cycle->lock_file;
711
712 if (lock_file.len) {
713 lock_file.len--;
714
715 if (ccf->lock_file.len != lock_file.len
716 || ngx_strncmp(ccf->lock_file.data, lock_file.data, lock_file.len)
717 != 0)
718 {
719 ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
720 "\"lock_file\" could not be changed, ignored");
721 }
722
723 cycle->lock_file.len = lock_file.len + 1;
724 lock_file.len += sizeof(".accept");
725
726 cycle->lock_file.data = ngx_pstrdup(cycle->pool, &lock_file);
727 if (cycle->lock_file.data == NULL) {
728 return NGX_CONF_ERROR;
729 }
730
731 } else {
732 cycle->lock_file.len = ccf->lock_file.len + 1;
733 cycle->lock_file.data = ngx_palloc(cycle->pool,
734 ccf->lock_file.len + sizeof(".accept"));
735 if (cycle->lock_file.data == NULL) {
736 return NGX_CONF_ERROR;
737 }
738
739 ngx_memcpy(ngx_cpymem(cycle->lock_file.data, ccf->lock_file.data,
740 ccf->lock_file.len),
741 ".accept", sizeof(".accept"));
706742 }
707743
708744 #endif
6363 #include <ngx_radix_tree.h>
6464 #include <ngx_times.h>
6565 #include <ngx_shmtx.h>
66 #include <ngx_slab.h>
6667 #if (NGX_OPENSSL)
6768 #include <ngx_event_openssl.h>
6869 #endif
4242 ngx_init_cycle(ngx_cycle_t *old_cycle)
4343 {
4444 void *rv;
45 u_char *lock_file;
4546 ngx_uint_t i, n;
4647 ngx_log_t *log;
4748 ngx_conf_t conf;
4849 ngx_pool_t *pool;
4950 ngx_cycle_t *cycle, **old;
50 ngx_list_part_t *part;
51 ngx_shm_zone_t *shm, *oshm;
52 ngx_slab_pool_t *shpool;
53 ngx_list_part_t *part, *opart;
5154 ngx_open_file_t *file;
5255 ngx_listening_t *ls, *nls;
5356 ngx_core_conf_t *ccf;
119122 }
120123
121124
125 if (old_cycle->shared_memory.part.nelts) {
126 n = old_cycle->shared_memory.part.nelts;
127 for (part = old_cycle->shared_memory.part.next; part; part = part->next)
128 {
129 n += part->nelts;
130 }
131
132 } else {
133 n = 1;
134 }
135
136 if (ngx_list_init(&cycle->shared_memory, pool, n, sizeof(ngx_shm_zone_t))
137 == NGX_ERROR)
138 {
139 ngx_destroy_pool(pool);
140 return NULL;
141 }
142
143
122144 cycle->new_log = ngx_log_create_errlog(cycle, NULL);
123145 if (cycle->new_log == NULL) {
124146 ngx_destroy_pool(pool);
180202 ngx_destroy_pool(pool);
181203 return NULL;
182204 }
183
184
185 #if 0
186 cycle->shm.size = /* STUB */ ngx_pagesize;
187 cycle->shm.log = log;
188
189 if (ngx_shm_alloc(&cycle->shm) != NGX_OK) {
190 ngx_destroy_pool(conf.temp_pool);
191 ngx_destroy_pool(pool);
192 return NULL;
193 }
194
195 cycle->shm_last = cycle->shm.addr;
196 cycle->shm_end = cycle->shm.addr + cycle->shm.size;
197 #endif
198205
199206
200207 conf.ctx = cycle->conf_ctx;
273280 #endif
274281
275282
276 if (ngx_test_lockfile(ccf->lock_file.data, log) != NGX_OK) {
283 if (ngx_test_lockfile(cycle->lock_file.data, log) != NGX_OK) {
277284 goto failed;
278285 }
279286
339346
340347 if (cycle->log->log_level == 0) {
341348 cycle->log->log_level = NGX_LOG_ERR;
349 }
350
351
352 /* create shared memory */
353
354 part = &cycle->shared_memory.part;
355 shm = part->elts;
356
357 for (i = 0; /* void */ ; i++) {
358
359 if (i >= part->nelts) {
360 if (part->next == NULL) {
361 break;
362 }
363 part = part->next;
364 shm = part->elts;
365 i = 0;
366 }
367
368 shm[i].shm.log = cycle->log;
369
370 opart = &old_cycle->shared_memory.part;
371 oshm = opart->elts;
372
373 for (n = 0; /* void */ ; n++) {
374
375 if (n >= opart->nelts) {
376 if (opart->next == NULL) {
377 break;
378 }
379 opart = opart->next;
380 oshm = opart->elts;
381 n = 0;
382 }
383
384 if (ngx_strcmp(shm[i].name.data, oshm[n].name.data) != 0) {
385 continue;
386 }
387
388 if (shm[i].shm.size == oshm[n].shm.size) {
389 shm[i].shm.addr = oshm[n].shm.addr;
390 goto found;
391 }
392
393 ngx_shm_free(&oshm[n].shm);
394
395 break;
396 }
397
398 if (ngx_shm_alloc(&shm[i].shm) != NGX_OK) {
399 goto failed;
400 }
401
402 shpool = (ngx_slab_pool_t *) shm[i].shm.addr;
403
404 shpool->end = shm[i].shm.addr + shm[i].shm.size;
405 shpool->min_shift = 3;
406
407 #if (NGX_HAVE_ATOMIC_OPS)
408
409 lock_file = NULL;
410
411 #else
412
413 lock_file = ngx_palloc(cycle->pool,
414 cycle->lock_file.len + shm[i].name.len);
415
416 if (lock_file == NULL) {
417 goto failed;
418 }
419
420 (void) ngx_cpystrn(ngx_cpymem(lock_file, cycle->lock_file.data,
421 cycle->lock_file.len),
422 shm[i].name.data, shm[i].name.len + 1);
423
424 #endif
425
426 if (ngx_shmtx_create(&shpool->mutex, (void *) &shpool->lock, lock_file)
427 != NGX_OK)
428 {
429 goto failed;
430 }
431
432 ngx_slab_init(shpool);
433
434 found:
435
436 continue;
342437 }
343438
344439
520615
521616 if (ngx_process == NGX_PROCESS_MASTER || ngx_is_init_cycle(old_cycle)) {
522617
523 if (old_cycle->shm.addr) {
524 ngx_shm_free(&old_cycle->shm);
525 }
526
527618 ngx_destroy_pool(old_cycle->pool);
528
529619 cycle->old_cycle = NULL;
530620
531621 return cycle;
629719 static void
630720 ngx_destroy_cycle_pools(ngx_conf_t *conf)
631721 {
632 if (conf->cycle->shm.addr) {
633 ngx_shm_free(&conf->cycle->shm);
634 }
635
636722 ngx_destroy_pool(conf->temp_pool);
637723 ngx_destroy_pool(conf->pool);
638724 }
2020 #define NGX_DEBUG_POINTS_ABORT 2
2121
2222
23 typedef struct {
24 ngx_shm_t shm;
25 ngx_str_t name;
26 } ngx_shm_zone_t;
27
28
2329 struct ngx_cycle_s {
2430 void ****conf_ctx;
2531 ngx_pool_t *pool;
3137 ngx_connection_t *free_connections;
3238 ngx_uint_t free_connection_n;
3339
34 ngx_shm_t shm;
35 u_char *shm_last;
36 u_char *shm_end;
37
3840 ngx_array_t listening;
3941 ngx_array_t pathes;
4042 ngx_list_t open_files;
43 ngx_list_t shared_memory;
4144
4245 ngx_uint_t connection_n;
4346 ngx_uint_t files_n;
5053
5154 ngx_str_t conf_file;
5255 ngx_str_t root;
56 ngx_str_t lock_file;
5357 };
5458
5559
189189 #endif
190190
191191
192 void
193 ngx_log_abort(ngx_err_t err, const char *text)
194 {
195 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, err, text);
196 }
197
198
192199 ngx_log_t *
193200 ngx_log_init(void)
194201 {
200200 ngx_log_t *ngx_log_init(void);
201201 ngx_log_t *ngx_log_create_errlog(ngx_cycle_t *cycle, ngx_array_t *args);
202202 char *ngx_set_error_log_levels(ngx_conf_t *cf, ngx_log_t *log);
203 void ngx_log_abort(ngx_err_t err, const char *text);
203204
204205
205206 extern ngx_module_t ngx_errlog_module;
192192 }
193193
194194
195 void *
196 ngx_shalloc(size_t size)
197 {
198 u_char *p;
199
200 if (size < sizeof(int) || (size & 1)) {
201 p = ngx_cycle->shm_last;
202
203 } else {
204 p = ngx_align_ptr(ngx_cycle->shm_last, NGX_ALIGNMENT);
205 }
206
207 if ((size_t) (ngx_cycle->shm_end - p) >= size) {
208 ngx_cycle->shm_last = p + size;
209 return p;
210 }
211
212 ngx_log_error(NGX_LOG_EMERG, ngx_cycle->log, 0,
213 "allocation of %uz bytes in shared memory failed, "
214 "only %uz are available",
215 size, ngx_cycle->shm_end - ngx_cycle->shm_last);
216
217 return NULL;
218 }
219
220
221 void *
222 ngx_shcalloc(size_t size)
223 {
224 void *p;
225
226 p = ngx_shalloc(size);
227 if (p) {
228 ngx_memzero(p, size);
229 }
230
231 return p;
232 }
233
234
235195 ngx_pool_cleanup_t *
236196 ngx_pool_cleanup_add(ngx_pool_t *p, size_t size)
237197 {
7070 void *ngx_pcalloc(ngx_pool_t *pool, size_t size);
7171 ngx_int_t ngx_pfree(ngx_pool_t *pool, void *p);
7272
73 void *ngx_shalloc(size_t size);
74 void *ngx_shcalloc(size_t size);
75 void ngx_shfree(void *p);
76
7773
7874 ngx_pool_cleanup_t *ngx_pool_cleanup_add(ngx_pool_t *p, size_t size);
7975 void ngx_pool_cleanup_file(void *data);
1111
1212
1313 ngx_int_t
14 ngx_shmtx_create(ngx_shmtx_t *mtx, void *addr, u_char *name, ngx_log_t *log)
14 ngx_shmtx_create(ngx_shmtx_t *mtx, void *addr, u_char *name)
1515 {
1616 mtx->lock = addr;
1717
2222
2323
2424 ngx_int_t
25 ngx_shmtx_create(ngx_shmtx_t *mtx, void *addr, u_char *name, ngx_log_t *log)
25 ngx_shmtx_create(ngx_shmtx_t *mtx, void *addr, u_char *name)
2626 {
2727 if (mtx->name) {
2828
2929 if (ngx_strcmp(name, mtx->name) == 0) {
3030 mtx->name = name;
31 mtx->log = log;
32
3331 return NGX_OK;
3432 }
3533
3937 mtx->fd = ngx_open_file(name, NGX_FILE_RDWR, NGX_FILE_CREATE_OR_OPEN);
4038
4139 if (mtx->fd == NGX_INVALID_FILE) {
42 ngx_log_error(NGX_LOG_EMERG, log, ngx_errno,
40 ngx_log_error(NGX_LOG_EMERG, ngx_cycle->log, ngx_errno,
4341 ngx_open_file_n " \"%s\" failed", name);
4442 return NGX_ERROR;
4543 }
4644
4745 if (ngx_delete_file(name) == NGX_FILE_ERROR) {
48 ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
46 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, ngx_errno,
4947 ngx_delete_file_n " \"%s\" failed", name);
5048 }
5149
5250 mtx->name = name;
53 mtx->log = log;
5451
5552 return NGX_OK;
5653 }
6057 ngx_shmtx_destory(ngx_shmtx_t *mtx)
6158 {
6259 if (ngx_close_file(mtx->fd) == NGX_FILE_ERROR) {
63 ngx_log_error(NGX_LOG_ALERT, mtx->log, ngx_errno,
60 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, ngx_errno,
6461 ngx_close_file_n " \"%s\" failed", mtx->name);
6562 }
6663 }
1717 #else
1818 ngx_fd_t fd;
1919 u_char *name;
20 ngx_log_t *log;
2120 #endif
2221 } ngx_shmtx_t;
2322
2423
25 ngx_int_t ngx_shmtx_create(ngx_shmtx_t *mtx, void *addr, u_char *name,
26 ngx_log_t *log);
24 ngx_int_t ngx_shmtx_create(ngx_shmtx_t *mtx, void *addr, u_char *name);
2725
2826
2927 #if (NGX_HAVE_ATOMIC_OPS)
6260 return 0;
6361 }
6462
65 ngx_log_error(NGX_LOG_ALERT, mtx->log, err, ngx_trylock_fd_n " failed");
66
67 ngx_abort();
63 ngx_log_abort(err, ngx_trylock_fd_n " failed");
6864 }
6965
7066
7975 return;
8076 }
8177
82 ngx_log_error(NGX_LOG_ALERT, mtx->log, err, ngx_lock_fd_n " failed");
83
84 ngx_abort();
78 ngx_log_abort(err, ngx_lock_fd_n " failed");
8579 }
8680
8781
9690 return;
9791 }
9892
99 ngx_log_error(NGX_LOG_ALERT, mtx->log, err, ngx_unlock_fd_n " failed");
100
101 ngx_abort();
93 ngx_log_abort(err, ngx_unlock_fd_n " failed");
10294 }
10395
10496
22 * Copyright (C) Igor Sysoev
33 */
44
5
6
7 void *ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size)
5 #include <ngx_config.h>
6 #include <ngx_core.h>
7
8 /*
9
10 12
11 2048 2 11
12 1024 4 10
13 512 8 9
14 256 16 8
15
16 128 32 4 32 7
17
18 64 64 8 63 6 1
19 32 128 16 127 5 1
20 16 256 32 254 4 2
21 8 512 64 504 3 8
22
23 */
24
25
26 #define NGX_SLAB_PAGE_MASK 3
27 #define NGX_SLAB_PAGE 0
28 #define NGX_SLAB_BIG 1
29 #define NGX_SLAB_EXACT 2
30 #define NGX_SLAB_SMALL 3
31
32 #if (NGX_PTR_SIZE == 4)
33
34 #define NGX_SLAB_PAGE_FREE 0
35 #define NGX_SLAB_PAGE_BUSY 0xffffffff
36 #define NGX_SLAB_PAGE_START 0x80000000
37
38 #define NGX_SLAB_SHIFT_MASK 0x0000000f
39 #define NGX_SLAB_MAP_MASK 0xffff0000
40 #define NGX_SLAB_MAP_SHIFT 16
41
42 #define NGX_SLAB_BUSY 0xffffffff
43
44 #else /* (NGX_PTR_SIZE == 8) */
45
46 #define NGX_SLAB_PAGE_FREE 0
47 #define NGX_SLAB_PAGE_BUSY 0xffffffffffffffff
48 #define NGX_SLAB_PAGE_START 0x8000000000000000
49
50 #define NGX_SLAB_SHIFT_MASK 0x000000000000000f
51 #define NGX_SLAB_MAP_MASK 0xffffffff00000000
52 #define NGX_SLAB_MAP_SHIFT 32
53
54 #define NGX_SLAB_BUSY 0xffffffffffffffff
55
56 #endif
57
58
59 #if (NGX_DEBUG_MALLOC)
60 #define ngx_slab_junk(p, size) ngx_memset(p, 0xD0, size)
61 #else
62 #define ngx_slab_junk(p, size)
63 #endif
64
65 static ngx_slab_page_t *ngx_slab_alloc_pages(ngx_slab_pool_t *pool,
66 ngx_uint_t pages);
67 static void ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
68 ngx_uint_t pages);
69
70
71 static ngx_uint_t ngx_slab_max_size;
72 static ngx_uint_t ngx_slab_exact_size;
73 static ngx_uint_t ngx_slab_exact_shift;
74
75
76 void
77 ngx_slab_init(ngx_slab_pool_t *pool)
878 {
9 return NULL;
79 u_char *p;
80 size_t size;
81 ngx_int_t m;
82 ngx_uint_t i, n, pages;
83 ngx_slab_page_t *slots;
84
85 /* STUB */
86 if (ngx_slab_max_size == 0) {
87 ngx_slab_max_size = ngx_pagesize / 2;
88 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t));
89 for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) {
90 /* void */
91 }
92 }
93 /**/
94
95 pool->min_size = 1 << pool->min_shift;
96
97 p = (u_char *) pool + sizeof(ngx_slab_pool_t);
98 size = pool->end - p;
99
100 ngx_slab_junk(p, size);
101
102 slots = (ngx_slab_page_t *) p;
103 n = ngx_pagesize_shift - pool->min_shift;
104
105 for (i = 0; i < n; i++) {
106 slots[i].slab = 0;
107 slots[i].next = &slots[i];
108 slots[i].prev = 0;
109 }
110
111 p += n * sizeof(ngx_slab_page_t);
112
113 /* STUB: possible overflow on 64-bit platform */
114 pages = (ngx_uint_t) ((uint64_t) size * ngx_pagesize
115 / (ngx_pagesize + sizeof(ngx_slab_page_t))
116 / ngx_pagesize);
117
118 ngx_memzero(p, pages * sizeof(ngx_slab_page_t));
119
120 pool->pages = (ngx_slab_page_t *) p;
121
122 pool->free.prev = 0;
123 pool->free.next = (ngx_slab_page_t *) p;
124
125 pool->pages->slab = pages;
126 pool->pages->next = &pool->free;
127 pool->pages->prev = (uintptr_t) &pool->free;
128
129 pool->start = (u_char *)
130 ngx_align((uintptr_t) p + pages * sizeof(ngx_slab_page_t),
131 ngx_pagesize);
132
133 m = pages - (pool->end - pool->start) / ngx_pagesize;
134 if (m > 0) {
135 pages -= m;
136 pool->pages->slab = pages;
137 }
138
139 #if 0
140 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, "slab: %p, %p, %ui, %d",
141 pool, pool->start, pages,
142 (pool->end - pool->start) / ngx_pagesize - pages);
143 #endif
10144 }
145
146
147 void *
148 ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size)
149 {
150 size_t s;
151 uintptr_t p, mask, *bitmap;
152 ngx_uint_t i, n, m, slot, shift, map;
153 ngx_slab_page_t *page, *prev, *slots;
154
155 ngx_shmtx_lock(&pool->mutex);
156
157 if (size >= ngx_slab_max_size) {
158 page = ngx_slab_alloc_pages(pool, (size + ngx_pagesize - 1)
159 >> ngx_pagesize_shift);
160 if (page) {
161 p = (page - pool->pages) << ngx_pagesize_shift;
162 p += (uintptr_t) pool->start;
163
164 } else {
165 p = 0;
166 }
167
168 goto done;
169 }
170
171 if (size > pool->min_size) {
172 shift = 1;
173 for (s = size - 1; s >>= 1; shift++) { /* void */ }
174 slot = shift - pool->min_shift;
175
176 } else {
177 size = pool->min_size;
178 shift = pool->min_shift;
179 slot = 0;
180 }
181
182 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
183 "slab alloc: %uz slot: %ui", size, slot);
184
185 slots = (ngx_slab_page_t *) ((u_char *) pool + sizeof(ngx_slab_pool_t));
186 page = slots[slot].next;
187
188 #if 0
189 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
190 "slab alloc: page %p next: %p", page, page->next);
191 #endif
192
193 if (page->next != page) {
194
195 if (size < ngx_slab_exact_size) {
196
197 do {
198 p = (page - pool->pages) << ngx_pagesize_shift;
199 bitmap = (uintptr_t *) (pool->start + p);
200
201 map = (1 << (ngx_pagesize_shift - shift))
202 / (sizeof(uintptr_t) * 8);
203
204 for (n = 0; n < map; n++) {
205
206 if (bitmap[n] != NGX_SLAB_BUSY) {
207
208 for (m = 1, i = 0; m; m <<= 1, i++) {
209 if ((bitmap[n] & m)) {
210 continue;
211 }
212
213 bitmap[n] |= m;
214 i <<= shift;
215
216 if (bitmap[n] == NGX_SLAB_BUSY) {
217 for (n = n + 1; n < map; n++) {
218 if (bitmap[n] != NGX_SLAB_BUSY) {
219 p = (uintptr_t) bitmap + i;
220
221 goto done;
222 }
223 }
224
225 prev = (ngx_slab_page_t *)
226 (page->prev & ~NGX_SLAB_PAGE_MASK);
227 prev->next = page->next;
228 page->next->prev = page->prev;
229
230 page->next = NULL;
231 page->prev = NGX_SLAB_SMALL;
232 }
233
234 p = (uintptr_t) bitmap + i;
235
236 goto done;
237 }
238 }
239 }
240
241 page = page->next;
242
243 } while (page);
244
245 } else if (size == ngx_slab_exact_size) {
246
247 do {
248 if (page->slab != NGX_SLAB_BUSY) {
249
250 for (m = 1, i = 0; m; m <<= 1, i++) {
251 if ((page->slab & m)) {
252 continue;
253 }
254
255 page->slab |= m;
256
257 if (page->slab == NGX_SLAB_BUSY) {
258 prev = (ngx_slab_page_t *)
259 (page->prev & ~NGX_SLAB_PAGE_MASK);
260 prev->next = page->next;
261 page->next->prev = page->prev;
262
263 page->next = NULL;
264 page->prev = NGX_SLAB_EXACT;
265 }
266
267 p = (page - pool->pages) << ngx_pagesize_shift;
268 p += i << shift;
269 p += (uintptr_t) pool->start;
270
271 goto done;
272 }
273 }
274
275 page = page->next;
276
277 } while (page);
278
279 } else { /* size < ngx_pagesize */
280
281 n = ngx_pagesize_shift - (page->slab & NGX_SLAB_SHIFT_MASK);
282 n = 1 << n;
283 n = (1 << n) - 1;
284 mask = n << NGX_SLAB_MAP_SHIFT;
285
286 do {
287 if ((page->slab & NGX_SLAB_MAP_MASK) != mask) {
288
289 for (m = 1 << NGX_SLAB_MAP_SHIFT, i = 0;
290 m & mask;
291 m <<= 1, i++)
292 {
293 if ((page->slab & m)) {
294 continue;
295 }
296
297 page->slab |= m;
298
299 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) {
300 prev = (ngx_slab_page_t *)
301 (page->prev & ~NGX_SLAB_PAGE_MASK);
302 prev->next = page->next;
303 page->next->prev = page->prev;
304
305 page->next = NULL;
306 page->prev = NGX_SLAB_BIG;
307 }
308
309 p = (page - pool->pages) << ngx_pagesize_shift;
310 p += i << shift;
311 p += (uintptr_t) pool->start;
312
313 goto done;
314 }
315 }
316
317 page = page->next;
318
319 } while (page);
320 }
321 }
322
323 page = ngx_slab_alloc_pages(pool, 1);
324
325 if (page) {
326 if (size < ngx_slab_exact_size) {
327 p = (page - pool->pages) << ngx_pagesize_shift;
328 bitmap = (uintptr_t *) (pool->start + p);
329
330 s = 1 << shift;
331 n = (1 << (ngx_pagesize_shift - shift)) / 8 / s;
332
333 if (n == 0) {
334 n = 1;
335 }
336
337 bitmap[0] = (2 << n) - 1;
338
339 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8);
340
341 for (i = 1; i < map; i++) {
342 bitmap[i] = 0;
343 }
344
345 page->slab = shift;
346 page->next = &slots[slot];
347 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
348
349 slots[slot].next = page;
350
351 p = ((page - pool->pages) << ngx_pagesize_shift) + s * n;
352 p += (uintptr_t) pool->start;
353
354 goto done;
355
356 } else if (size == ngx_slab_exact_size) {
357
358 page->slab = 1;
359 page->next = &slots[slot];
360 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
361
362 slots[slot].next = page;
363
364 p = (page - pool->pages) << ngx_pagesize_shift;
365 p += (uintptr_t) pool->start;
366
367 goto done;
368
369 } else { /* size < ngx_pagesize */
370
371 page->slab = (1 << NGX_SLAB_MAP_SHIFT) | shift;
372 page->next = &slots[slot];
373 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
374
375 slots[slot].next = page;
376
377 p = (page - pool->pages) << ngx_pagesize_shift;
378 p += (uintptr_t) pool->start;
379
380 goto done;
381 }
382 }
383
384 p = 0;
385
386 done:
387
388 ngx_shmtx_unlock(&pool->mutex);
389
390 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab alloc: %p", p);
391
392 return (void *) p;
393 }
394
395
396 void
397 ngx_slab_free(ngx_slab_pool_t *pool, void *p)
398 {
399 size_t size;
400 uintptr_t slab, *bitmap;
401 ngx_uint_t n, m, type, slot, shift, map;
402 ngx_slab_page_t *slots, *page;
403
404 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p);
405
406 ngx_shmtx_lock(&pool->mutex);
407
408 if ((u_char *) p < pool->start || (u_char *) p > pool->end) {
409 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
410 "ngx_slab_free(): outside of pool");
411 goto fail;
412 }
413
414 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
415 page = &pool->pages[n];
416 slab = page->slab;
417 type = page->prev & NGX_SLAB_PAGE_MASK;
418
419 switch (type) {
420
421 case NGX_SLAB_SMALL:
422
423 shift = slab & NGX_SLAB_SHIFT_MASK;
424 size = 1 << shift;
425
426 if ((uintptr_t) p & (size - 1)) {
427 goto wrong_chunk;
428 }
429
430 n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift;
431 m = 1 << (n & (sizeof(uintptr_t) * 8 - 1));
432 n /= (sizeof(uintptr_t) * 8);
433 bitmap = (uintptr_t *) ((uintptr_t) p & ~(ngx_pagesize - 1));
434
435 if (bitmap[n] & m) {
436
437 if (page->next == NULL) {
438 slots = (ngx_slab_page_t *)
439 ((u_char *) pool + sizeof(ngx_slab_pool_t));
440 slot = shift - pool->min_shift;
441
442 page->next = slots[slot].next;
443 slots[slot].next = page;
444
445 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
446 page->next->prev = (uintptr_t) page | NGX_SLAB_SMALL;
447 }
448
449 bitmap[n] &= ~m;
450
451 n = (1 << (ngx_pagesize_shift - shift)) / 8 / (1 << shift);
452
453 if (n == 0) {
454 n = 1;
455 }
456
457 if (bitmap[0] & ~((1 << n) - 1)) {
458 goto done;
459 }
460
461 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8);
462
463 for (n = 1; n < map; n++) {
464 if (bitmap[n]) {
465 goto done;
466 }
467 }
468
469 ngx_slab_free_pages(pool, page, 1);
470
471 goto done;
472 }
473
474 goto chunk_already_free;
475
476 case NGX_SLAB_EXACT:
477
478 m = 1 << (((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift);
479 size = ngx_slab_exact_size;
480
481 if ((uintptr_t) p & (size - 1)) {
482 goto wrong_chunk;
483 }
484
485 if (slab & m) {
486 if (slab == NGX_SLAB_BUSY) {
487 slots = (ngx_slab_page_t *)
488 ((u_char *) pool + sizeof(ngx_slab_pool_t));
489 slot = ngx_slab_exact_shift - pool->min_shift;
490
491 page->next = slots[slot].next;
492 slots[slot].next = page;
493
494 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
495 page->next->prev = (uintptr_t) page | NGX_SLAB_EXACT;
496 }
497
498 page->slab &= ~m;
499
500 if (page->slab) {
501 goto done;
502 }
503
504 ngx_slab_free_pages(pool, page, 1);
505
506 goto done;
507 }
508
509 goto chunk_already_free;
510
511 case NGX_SLAB_BIG:
512
513 shift = slab & NGX_SLAB_SHIFT_MASK;
514 size = 1 << shift;
515
516 if ((uintptr_t) p & (size - 1)) {
517 goto wrong_chunk;
518 }
519
520 m = 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift)
521 + NGX_SLAB_MAP_SHIFT);
522
523 if (slab & m) {
524
525 if (page->next == NULL) {
526 slots = (ngx_slab_page_t *)
527 ((u_char *) pool + sizeof(ngx_slab_pool_t));
528 slot = shift - pool->min_shift;
529
530 page->next = slots[slot].next;
531 slots[slot].next = page;
532
533 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
534 page->next->prev = (uintptr_t) page | NGX_SLAB_BIG;
535 }
536
537 page->slab &= ~m;
538
539 if (page->slab & NGX_SLAB_MAP_MASK) {
540 goto done;
541 }
542
543 ngx_slab_free_pages(pool, page, 1);
544
545 goto done;
546 }
547
548 goto chunk_already_free;
549
550 case NGX_SLAB_PAGE:
551
552 if ((uintptr_t) p & (ngx_pagesize - 1)) {
553 goto wrong_chunk;
554 }
555
556 if (slab == NGX_SLAB_PAGE_FREE) {
557 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
558 "ngx_slab_free(): page is already free");
559 goto fail;
560 }
561
562 if (slab == NGX_SLAB_PAGE_BUSY) {
563 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
564 "ngx_slab_free(): pointer to wrong page");
565 goto fail;
566 }
567
568 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
569 size = slab & ~NGX_SLAB_PAGE_START;
570
571 ngx_slab_free_pages(pool, &pool->pages[n], size);
572
573 size <<= ngx_pagesize_shift;
574
575 goto done;
576 }
577
578 /* not reached */
579
580 return;
581
582 done:
583
584 ngx_slab_junk(p, size);
585
586 ngx_shmtx_unlock(&pool->mutex);
587
588 return;
589
590 wrong_chunk:
591
592 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
593 "ngx_slab_free(): pointer to wrong chunk");
594
595 goto fail;
596
597 chunk_already_free:
598
599 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
600 "ngx_slab_free(): chunk is already free");
601
602 fail:
603
604 ngx_shmtx_unlock(&pool->mutex);
605
606 return;
607 }
608
609
610 static ngx_slab_page_t *
611 ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages)
612 {
613 ngx_slab_page_t *page, *p;
614
615 for (page = pool->free.next; page != &pool->free; page = page->next) {
616
617 if (page->slab >= pages) {
618
619 if (page->slab > pages) {
620 page[pages].slab = page->slab - pages;
621 page[pages].next = page->next;
622 page[pages].prev = page->prev;
623
624 p = (ngx_slab_page_t *) page->prev;
625 p->next = &page[pages];
626 page->next->prev = (uintptr_t) &page[pages];
627
628 } else {
629 p = (ngx_slab_page_t *) page->prev;
630 p->next = page->next;
631 page->next->prev = page->prev;
632 }
633
634 page->slab = pages | NGX_SLAB_PAGE_START;
635
636 #if (NGX_DEBUG)
637 page->next = NULL;
638 page->prev = NGX_SLAB_PAGE;
639 #endif
640
641 if (--pages == 0) {
642 return page;
643 }
644
645 for (p = page + 1; pages; pages--) {
646 p->slab = NGX_SLAB_PAGE_BUSY;
647 #if (NGX_DEBUG)
648 p->next = NULL;
649 p->prev = NGX_SLAB_PAGE;
650 #endif
651 p++;
652 }
653
654 return page;
655 }
656 }
657
658 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, NGX_ENOMEM,
659 "ngx_slab_alloc(): failed");
660 return NULL;
661 }
662
663
664 static void
665 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
666 ngx_uint_t pages)
667 {
668 ngx_slab_page_t *prev;
669
670 page->slab = pages--;
671
672 if (pages) {
673 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t));
674 }
675
676 prev = (ngx_slab_page_t *) (page->prev & ~NGX_SLAB_PAGE_MASK);
677 prev->next = page->next;
678
679 page->next = pool->free.next;
680 pool->free.next = page;
681
682 page->prev = page->next->prev;
683 page->next->prev = (uintptr_t) page;
684 }
1111 #include <ngx_core.h>
1212
1313
14 typedef struct ngx_free_slab_s ngx_free_slab_t;
14 typedef struct ngx_slab_page_s ngx_slab_page_t;
1515
16 typedef struct ngx_free_slab_s {
17 ngx_free_slab_t *next;
18 }
19
20
21 typedef struct ngx_slab_block_s ngx_slab_block_t;
22
23 typedef struct ngx_slab_block_s {
24 ngx_free_slab_t *free;
25 ngx_slab_buf_t *next;
26 size_t color;
16 struct ngx_slab_page_s {
17 uintptr_t slab;
18 ngx_slab_page_t *next;
19 uintptr_t prev;
2720 };
2821
2922
3023 typedef struct {
31 ngx_slab_buf_t *blocks;
32 size_t size;
24 ngx_atomic_t lock;
3325
34 void *start;
35 uint32_t map;
26 size_t min_size;
27 size_t min_shift;
3628
37 ngx_log_t *log;
38 ngx_free_pool_t free;
29 ngx_slab_page_t *pages;
30 ngx_slab_page_t free;
31
32 u_char *start;
33 u_char *end;
34
35 ngx_shmtx_t mutex;
3936 } ngx_slab_pool_t;
4037
4138
39 void ngx_slab_init(ngx_slab_pool_t *pool);
40 void *ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size);
41 void ngx_slab_free(ngx_slab_pool_t *pool, void *p);
42
43
4244 #endif /* _NGX_SLAB_H_INCLUDED_ */
482482 return NGX_OK;
483483 }
484484
485
486485 if (ngx_accept_mutex_ptr) {
487
488 /* reinit ngx_accept_mutex */
489
490 if (ngx_shmtx_create(&ngx_accept_mutex, (void *) ngx_accept_mutex_ptr,
491 ccf->lock_file.data, cycle->log)
492 != NGX_OK)
493 {
494 return NGX_ERROR;
495 }
496
497486 return NGX_OK;
498487 }
499488
527516
528517 ngx_accept_mutex_ptr = (ngx_atomic_t *) shared;
529518
530 if (ngx_shmtx_create(&ngx_accept_mutex, shared, ccf->lock_file.data,
531 cycle->log)
519 if (ngx_shmtx_create(&ngx_accept_mutex, shared, cycle->lock_file.data)
532520 != NGX_OK)
533521 {
534522 return NGX_ERROR;
77 #include <ngx_core.h>
88
99
10 int ngx_pagesize;
10 ngx_uint_t ngx_pagesize;
11 ngx_uint_t ngx_pagesize_shift;
1112 ngx_uint_t ngx_cacheline_size;
1213
1314
3535 #endif
3636
3737
38 extern int ngx_pagesize;
38 extern ngx_uint_t ngx_pagesize;
39 extern ngx_uint_t ngx_pagesize_shift;
3940 extern ngx_uint_t ngx_cacheline_size;
4041
4142
2929 ngx_int_t
3030 ngx_os_init(ngx_log_t *log)
3131 {
32 ngx_uint_t n;
33
3234 #if (NGX_HAVE_OS_SPECIFIC_INIT)
3335 if (ngx_os_specific_init(log) != NGX_OK) {
3436 return NGX_ERROR;
3941
4042 ngx_pagesize = getpagesize();
4143 ngx_cacheline_size = NGX_CPU_CACHE_LINE;
44
45 n = ngx_pagesize;
46
47 for (n = ngx_pagesize; n >>= 1; ngx_pagesize_shift++) { /* void */ }
4248
4349 if (ngx_ncpu == 0) {
4450 ngx_ncpu = 1;
77 #include <ngx_core.h>
88
99
10 int ngx_pagesize;
10 ngx_uint_t ngx_pagesize;
11 ngx_uint_t ngx_pagesize_shift;
1112 ngx_uint_t ngx_cacheline_size;
1213
1314
1717 #define ngx_free free
1818 #define ngx_memalign(alignment, size, log) ngx_alloc(size, log)
1919
20 extern int ngx_pagesize;
20 extern ngx_uint_t ngx_pagesize;
21 extern ngx_uint_t ngx_pagesize_shift;
2122 extern ngx_uint_t ngx_cacheline_size;
2223
2324
2020
2121 #define NGX_EPERM ERROR_ACCESS_DENIED
2222 #define NGX_ENOENT ERROR_FILE_NOT_FOUND
23 #define NGX_ENOMEM ERROR_NOT_ENOUGH_MEMORY
2324 #define NGX_EACCES ERROR_ACCESS_DENIED
2425 #if 0
2526 #define NGX_EEXIST ERROR_FILE_EXISTS
5252 DWORD bytes;
5353 SOCKET s;
5454 WSADATA wsd;
55 ngx_uint_t n;
5556 SYSTEM_INFO si;
5657
5758 /* get Windows version */
100101 ngx_ncpu = si.dwNumberOfProcessors;
101102 ngx_cacheline_size = NGX_CPU_CACHE_LINE;
102103
104 for (n = ngx_pagesize; n >>= 1; ngx_pagesize_shift++) { /* void */ }
105
103106
104107 /* init Winsock */
105108