Memcached源码分析之items.c

1. #include "memcached.h"
2. #include <sys/stat.h>
3. #include <sys/socket.h>
4. #include <sys/signal.h>
5. #include <sys/resource.h>
6. #include <fcntl.h>
7. #include <netinet/in.h>
8. #include <errno.h>
9. #include <stdlib.h>
10. #include <stdio.h>
11. #include <string.h>
12. #include <time.h>
13. #include <assert.h>
14. #include <unistd.h>
15. static void item_link_q(item *it);
16. static void item_unlink_q(item *it);
17. #define LARGEST_ID POWER_LARGEST
18. typedef struct {
19.     uint64_t evicted;
20.     uint64_t evicted_nonzero;
21.     rel_time_t evicted_time;
22.     uint64_t reclaimed;
23.     uint64_t outofmemory;
24.     uint64_t tailrepairs;
25.     uint64_t expired_unfetched;
26.     uint64_t evicted_unfetched;
27.     uint64_t crawler_reclaimed;
28. } itemstats_t;
29. static item *heads[LARGEST_ID]; //各个slabclass的LRU队列头指针数组
30. static item *tails[LARGEST_ID]; //各个slabclass的LRU队列尾指针数组
31. static crawler crawlers[LARGEST_ID]; //各个slabclass的item爬虫数组
32. static itemstats_t itemstats[LARGEST_ID]; //各个slabclass的item统计数组
33. static unsigned int sizes[LARGEST_ID]; //各个slabclass的chunk大小数组
34. static int crawler_count = 0;
35. static volatile int do_run_lru_crawler_thread = 0;
36. static int lru_crawler_initialized = 0;
37. static pthread_mutex_t lru_crawler_lock = PTHREAD_MUTEX_INITIALIZER;
38. static pthread_cond_t lru_crawler_cond = PTHREAD_COND_INITIALIZER;
39. //重置统计
40. void item_stats_reset(void) {
41.     mutex_lock(&cache_lock);
42.     memset(itemstats, 0, sizeof(itemstats));
43.     mutex_unlock(&cache_lock);
44. }
45. /* Get the next CAS id for a new item. */
46. uint64_t get_cas_id(void) {
47.     static uint64_t cas_id = 0;
48.     return ++cas_id;
49. }
50. /* Enable this for reference-count debugging. */
51. #if 0
52. # define DEBUG_REFCNT(it,op)
53.                 fprintf(stderr, "item %x refcnt(%c) %d %c%c%c ",
54.                         it, op, it->refcount,
55.                         (it->it_flags & ITEM_LINKED) ? 'L' : ' ',
56.                         (it->it_flags & ITEM_SLABBED) ? 'S' : ' ')
57. #else
58. # define DEBUG_REFCNT(it,op) while(0)
59. #endif
60. /**
61.     算出item总大小
62.  */
63. static size_t item_make_header(const uint8_t nkey, const int flags, const int nbytes,
64.                      char *suffix, uint8_t *nsuffix) {
65.     /* suffix is defined at 40 chars elsewhere.. */
66.     *nsuffix = (uint8_t) snprintf(suffix, 40, " %d %d ", flags, nbytes - 2);
67.     return sizeof(item) + nkey + *nsuffix + nbytes;
68. }
69. /**
70. item分配
71. 把这个函数弄清楚，基本就把memcached内存管理机制大体弄清楚了。
72. */
73. item *do_item_alloc(char *key, const size_t nkey, const int flags,
74.                     const rel_time_t exptime, const int nbytes,
75.                     const uint32_t cur_hv) {
76.     uint8_t nsuffix;
77.     item *it = NULL;
78.     char suffix[40];
79.     size_t ntotal = item_make_header(nkey + 1, flags, nbytes, suffix, &nsuffix); //item总大小
80.     if (settings.use_cas) {
81.         ntotal += sizeof(uint64_t); //如果有用到cas 那么item大小还要加上unit64_t的size
82.     }
83.     unsigned int id = slabs_clsid(ntotal); //根据item大小，找到适合的slabclass
84.     if (id == 0)
85.         return 0;
86.     mutex_lock(&cache_lock); //cache锁
87.     /* do a quick check if we have any expired items in the tail.. */
88.     /* 准备分配新的item了，随便快速瞄一下lru链表末尾有没有过期item，有的话就用过期的空间 */
89.     int tries = 5;
90.     int tried_alloc = 0;
91.     item *search;
92.     void *hold_lock = NULL;
93.     rel_time_t oldest_live = settings.oldest_live;
94.     search = tails[id]; //这个tails是一个全局变量，tails[xx]是id为xx的slabclass lru链表的尾部
95.     /* We walk up *only* for locked items. Never searching for expired.
96.      * Waste of CPU for almost all deployments */
97.     //从LRU链表尾部（就是最久没使用过的item）开始往前找
98.     for (; tries > 0 && search != NULL; tries--, search=search->prev) {
99.         if (search->nbytes == 0 && search->nkey == 0 && search->it_flags == 1) {
100.             /* We are a crawler, ignore it. */
101.             /*
102.                 这里注释意思是说我们现在是以爬虫的身份来爬出过期的空间，
103.                 像爬到这种很怪的item，就别管了，不是爬虫要做的事，不要就行了。
104.              */
105.             tries++;
106.             continue;
107.         }
108.         /**
109.         你会看到很多地方有这个hv，简单说下，其实它是对item的一个hash，得到hv值，这个hv主要有两个
110.         作用：
111.         1）用于hash表保存item，通过hv计算出哈希表中的桶号
112.         2）用于item lock表中锁住item，通过hv计算出应该用item lock表中哪个锁对当前item进行加锁
113.         这两者都涉及到一个粒度问题，不可能保证每个不一样的key的hv不会相同，所有hash方法都可能
114.         出现冲突。
115.         所以hash表中用链表的方式处理冲突的item，而item lock表中会多个item共享一个锁，或者说
116.         多个桶共享一个锁。
117.         */
118.         uint32_t hv = hash(ITEM_key(search), search->nkey);
119.          /**
120.          尝试去锁住当前item。
121.          */
122.         if (hv == cur_hv || (hold_lock = item_trylock(hv)) == NULL)
123.             continue;
124.         /* Now see if the item is refcount locked */
125.         if (refcount_incr(&search->refcount) != 2) {
126.             refcount_decr(&search->refcount);
127.             /* Old rare bug could cause a refcount leak. We haven't seen
128.              * it in years, but we leave this code in to prevent failures
129.              * just in case
130.             没看懂这里的意思.....
131.              */
132.             if (settings.tail_repair_time &&
133.                     search->time + settings.tail_repair_time < current_time) {
134.                 itemstats[id].tailrepairs++;
135.                 search->refcount = 1;
136.                 do_item_unlink_nolock(search, hv);
137.             }
138.             if (hold_lock)
139.                 item_trylock_unlock(hold_lock);
140.             continue;
141.         }
142.         /* Expired or flushed */
143.         //超时了...
144.         if ((search->exptime != 0 && search->exptime < current_time)
145.             || (search->time <= oldest_live && oldest_live <= current_time)) {
146.             itemstats[id].reclaimed++;
147.             if ((search->it_flags & ITEM_FETCHED) == 0) {
148.                 itemstats[id].expired_unfetched++;
149.             }
150.             it = search; //拿下空间
151.             slabs_adjust_mem_requested(it->slabs_clsid, ITEM_ntotal(it), ntotal); //更新统计数据
152.             /**
153.             什么是link，在这简单说下，就是把item加到哈希表和LRU链表的过程。详见items::do_item_link函数这里把item旧的link取消掉，当前函数do_item_alloc的工作只是拿空间，而往后可知道拿到item空间后会对这块item进行“link”工作，而这里这块item空间是旧的item超时然后拿来用的，所以先把它unlink掉
154.             */
155.             do_item_unlink_nolock(it, hv);
156.             /* Initialize the item block: */
157.             it->slabs_clsid = 0;
158.         } else if ((it = slabs_alloc(ntotal, id)) == NULL) {/*如果没有找到超时的item，则
159.                 调用slabs_alloc分配空间，详见slabs_alloc
160.                 如果slabs_alloc分配空间失败，即返回NULL，则往下走，下面的代码是
161.                 把LRU列表最后一个给淘汰，即使item没有过期。
162.                 这里一般是可用内存已经满了，需要按LRU进行淘汰的时候。
163.                 //************mark: $1**************
164.             */
165.             tried_alloc = 1; //标记一下，表示有进入此分支，表示有尝试过调用slabs_alloc去分配新的空间。
166.             //记下被淘汰item的信息，像我们使用memcached经常会查看的evicted_time就是在这里赋值啦！
167.             if (settings.evict_to_free == 0) {
168.                 itemstats[id].outofmemory++;
169.             } else {
170.                 itemstats[id].evicted++;
171.                 itemstats[id].evicted_time = current_time - search->time; //被淘汰的item距离上次使用多长时间了
172.                 if (search->exptime != 0)
173.                     itemstats[id].evicted_nonzero++;
174.                 if ((search->it_flags & ITEM_FETCHED) == 0) {
175.                     itemstats[id].evicted_unfetched++;
176.                 }
177.                 it = search;
178.                 slabs_adjust_mem_requested(it->slabs_clsid, ITEM_ntotal(it), ntotal);//更新统计数据
179.                 do_item_unlink_nolock(it, hv); //从哈希表和LRU链表中删掉
180.                 /* Initialize the item block: */
181.                 it->slabs_clsid = 0;
182.                 /*
183.                  如果当前slabclass有item被淘汰掉了，说明可用内存都满了，再也没有
184.                  slab可分配了，
185.                  而如果 slab_automove=2 (默认是1)，这样会导致angry模式，
186.                  就是只要分配失败了，就马上进行slab重分配：把别的slabclass空间牺牲
187.                  掉一些，马上给现在的slabclass分配空间，而不会合理地根据淘汰统计
188.                  数据来分析要怎么重分配（slab_automove = 1则会）。
189.                  */
190.                 if (settings.slab_automove == 2)
191.                     slabs_reassign(-1, id);
192.             }
193.         }
194.         refcount_decr(&search->refcount);
195.         /* If hash values were equal, we don't grab a second lock */
196.         if (hold_lock)
197.             item_trylock_unlock(hold_lock);
198.         break;
199.     }
200.     /**
201.     如果上面的for循环里面没有找到空间，并且没有进入过else if ((it = slabs_alloc(ntotal, id)) == NULL)这个分支没有
202.     尝试调slabs_alloc分配空间（有这种可能性），那么，下面这行代码就是再尝试分配。
203.     你会觉得上面那个循环写得特纠结，逻辑不清，估计你也看醉了。其实整个分配原则是这样子：
204.     1）先从LRU链表找下看看有没有恰好过期的空间，有的话就用这个空间。
205.     2）如果没有过期的空间，就分配新的空间。
206.     3）如果分配新的空间失败，那么往往是内存都用光了，则从LRU链表中把最旧的即使没过期的item淘汰掉，空间分给新的item用。
207.     问题是：这个从“LRU链表找到的item”是一个不确定的东西，有可能这个item数据异常，有可能这个item由于与别的item共用锁的桶号
208.     这个桶被锁住了，所以总之各种原因这个item此刻不一定可用，因此用了一个循环尝试找几次（上面是5）。
209.     所以逻辑是：
210.     1）我先找5次LRU看看有没有可用的过期的item，有就用它。（for循环5次）
211.     2）5次没有找到可用的过期的item，那我分配新的。
212.     3）分配新的不成功，那我再找5次看看有没有可用的虽然没过期的item，淘汰它，把空间给新的item用。（for循环5次）
213.     那么这里有个问题，如果代码要写得逻辑清晰一点，我得写两个for循环，一个是为了第2）步前“找可用的过期的”item，
214.     一个是第2）步不成功后“找可用的用来淘汰的”空间。而且有重复的逻辑“找到可用的”，所以memcached作者就合在一起了，
215.     然后只能把第2）步也塞到for循环里面，确实挺尴尬的。。。估计memcached作者也写得很纠结。。。
216.     所以就很有可能出现5次都没找到可用的空间，都没进入过elseif那个分支就被continue掉了，为了记下有没有进过elseif
217.     分支就挫挫地用一个tried_alloc变量来做记号。。
218.     */
219.     if (!tried_alloc && (tries == 0 || search == NULL))
220.         it = slabs_alloc(ntotal, id);
221.     if (it == NULL) {
222.         itemstats[id].outofmemory++;
223.         mutex_unlock(&cache_lock);
224.         return NULL; //没错！会有分配新空间不成功，而且尝试5次淘汰旧的item也没成功的时候，只能返回NULL。。
225.     }
226.     assert(it->slabs_clsid == 0);
227.     assert(it != heads[id]);
228.     //来到这里，说明item分配成功，下面主要是一些初始化工作。
229.     /* Item initialization can happen outside of the lock; the item's already
230.      * been removed from the slab LRU.
231.      */
232.     it->refcount = 1; /* the caller will have a reference */
233.     mutex_unlock(&cache_lock);
234.     it->next = it->prev = it->h_next = 0;
235.     it->slabs_clsid = id;
236.     DEBUG_REFCNT(it, '*');
237.     it->it_flags = settings.use_cas ? ITEM_CAS : 0;
238.     it->nkey = nkey;
239.     it->nbytes = nbytes;
240.     memcpy(ITEM_key(it), key, nkey);
241.     it->exptime = exptime;
242.     memcpy(ITEM_suffix(it), suffix, (size_t)nsuffix);
243.     it->nsuffix = nsuffix;
244.     return it;
245. }
246. /**
247. 把这块item free掉，以供再利用，注意这里的free不是指把内存空间释放哦，
248. 而是把这块item 变为“空闲”
249. */
250. void item_free(item *it) {
251.     size_t ntotal = ITEM_ntotal(it);
252.     unsigned int clsid;
253.     assert((it->it_flags & ITEM_LINKED) == 0);
254.     assert(it != heads[it->slabs_clsid]);
255.     assert(it != tails[it->slabs_clsid]);
256.     assert(it->refcount == 0);
257.     /* so slab size changer can tell later if item is already free or not */
258.     clsid = it->slabs_clsid;
259.     it->slabs_clsid = 0; //在这把free掉的item 的slabs_clsid设为0
260.     DEBUG_REFCNT(it, 'F');
261.     slabs_free(it, ntotal, clsid);
262. }
263. /**
264.  * 检查item大小
265.  */
266. bool item_size_ok(const size_t nkey, const int flags, const int nbytes) {
267.     char prefix[40];
268.     uint8_t nsuffix;
269.     size_t ntotal = item_make_header(nkey + 1, flags, nbytes,
270.                                      prefix, &nsuffix);
271.     if (settings.use_cas) {
272.         ntotal += sizeof(uint64_t);
273.     }
274.     return slabs_clsid(ntotal) != 0;
275. }
276. /**
277. 把item插入相应的slabclass lru链表中而已
278. */
279. static void item_link_q(item *it) { /* item is the new head */
280.     item **head, **tail;
281.     assert(it->slabs_clsid < LARGEST_ID);
282.     assert((it->it_flags & ITEM_SLABBED) == 0);
283.     head = &heads[it->slabs_clsid];
284.     tail = &tails[it->slabs_clsid];
285.     assert(it != *head);
286.     assert((*head && *tail) || (*head == 0 && *tail == 0));
287.     it->prev = 0;
288.     it->next = *head;
289.     if (it->next) it->next->prev = it;
290.     *head = it;
291.     if (*tail == 0) *tail = it;
292.     sizes[it->slabs_clsid]++;
293.     return;
294. }
295. /**
296. 把item从相应的slabclass lru链表中删掉而已，下面就是经典的删除链表逻辑代码了
297. */
298. static void item_unlink_q(item *it) {
299.     item **head, **tail;
300.     assert(it->slabs_clsid < LARGEST_ID);
301.     head = &heads[it->slabs_clsid];
302.     tail = &tails[it->slabs_clsid];
303.     if (*head == it) {
304.         assert(it->prev == 0);
305.         *head = it->next;
306.     }
307.     if (*tail == it) {
308.         assert(it->next == 0);
309.         *tail = it->prev;
310.     }
311.     assert(it->next != it);
312.     assert(it->prev != it);
313.     if (it->next) it->next->prev = it->prev;
314.     if (it->prev) it->prev->next = it->next;
315.     sizes[it->slabs_clsid]--;
316.     return;
317. }
318. /**
319. 把item "link"起来，主要包括：
320. 1）改变一些统计数据
321. 2）把item加到哈希表
322. 3）把item插入到相应的slabclass lru链表中
323. */
324. int do_item_link(item *it, const uint32_t hv) {
325.     MEMCACHED_ITEM_LINK(ITEM_key(it), it->nkey, it->nbytes);
326.     assert((it->it_flags & (ITEM_LINKED|ITEM_SLABBED)) == 0);
327.     mutex_lock(&cache_lock);
328.     it->it_flags |= ITEM_LINKED;
329.     it->time = current_time;
330.     STATS_LOCK();
331.     stats.curr_bytes += ITEM_ntotal(it);
332.     stats.curr_items += 1;
333.     stats.total_items += 1;
334.     STATS_UNLOCK();
335.     /* Allocate a new CAS ID on link. */
336.     ITEM_set_cas(it, (settings.use_cas) ? get_cas_id() : 0);
337.     assoc_insert(it, hv); //插入哈希表
338.     item_link_q(it); //加入LRU链表
339.     refcount_incr(&it->refcount);
340.     mutex_unlock(&cache_lock);
341.     return 1;
342. }
343. /**
344. 就是和do_item_link反过来的一些操作
345. */
346. void do_item_unlink(item *it, const uint32_t hv) {
347.     MEMCACHED_ITEM_UNLINK(ITEM_key(it), it->nkey, it->nbytes);
348.     mutex_lock(&cache_lock);
349.     if ((it->it_flags & ITEM_LINKED) != 0) {
350.         it->it_flags &= ~ITEM_LINKED;
351.         STATS_LOCK();
352.         stats.curr_bytes -= ITEM_ntotal(it);
353.         stats.curr_items -= 1;
354.         STATS_UNLOCK();
355.         assoc_delete(ITEM_key(it), it->nkey, hv);
356.         item_unlink_q(it);
357.         do_item_remove(it);
358.     }
359.     mutex_unlock(&cache_lock);
360. }
361. /* FIXME: Is it necessary to keep this copy/pasted code? */
362. void do_item_unlink_nolock(item *it, const uint32_t hv) {
363.     MEMCACHED_ITEM_UNLINK(ITEM_key(it), it->nkey, it->nbytes);
364.     if ((it->it_flags & ITEM_LINKED) != 0) {
365.         it->it_flags &= ~ITEM_LINKED;
366.         STATS_LOCK();
367.         stats.curr_bytes -= ITEM_ntotal(it);
368.         stats.curr_items -= 1;
369.         STATS_UNLOCK();
370.         assoc_delete(ITEM_key(it), it->nkey, hv);
371.         item_unlink_q(it);
372.         do_item_remove(it);
373.     }
374. }
375. /**
376. 指向item的指针不用的时候都会调用此函数
377. */
378. void do_item_remove(item *it) {
379.     MEMCACHED_ITEM_REMOVE(ITEM_key(it), it->nkey, it->nbytes);
380.     assert((it->it_flags & ITEM_SLABBED) == 0);
381.     assert(it->refcount > 0);
382.     if (refcount_decr(&it->refcount) == 0) { //引用计数减1，当引用计数为0时，才真正把item free掉。
383.         item_free(it);
384.     }
385. }
386. /**
387. 主要作用是重置在最近使用链表中的位置，更新最近使用时间
388. */
389. void do_item_update(item *it) {
390.     MEMCACHED_ITEM_UPDATE(ITEM_key(it), it->nkey, it->nbytes);
391.     if (it->time < current_time - ITEM_UPDATE_INTERVAL) {
392.         assert((it->it_flags & ITEM_SLABBED) == 0);
393.         mutex_lock(&cache_lock);
394.         if ((it->it_flags & ITEM_LINKED) != 0) {
395.             item_unlink_q(it);
396.             it->time = current_time;
397.             item_link_q(it);
398.         }
399.         mutex_unlock(&cache_lock);
400.     }
401. }
402. int do_item_replace(item *it, item *new_it, const uint32_t hv) {
403.     MEMCACHED_ITEM_REPLACE(ITEM_key(it), it->nkey, it->nbytes,
404.                            ITEM_key(new_it), new_it->nkey, new_it->nbytes);
405.     assert((it->it_flags & ITEM_SLABBED) == 0);
406.     do_item_unlink(it, hv);
407.     return do_item_link(new_it, hv);
408. }
409. void item_stats_evictions(uint64_t *evicted) {
410.     int i;
411.     mutex_lock(&cache_lock);
412.     for (i = 0; i < LARGEST_ID; i++) {
413.         evicted[i] = itemstats[i].evicted;
414.     }
415.     mutex_unlock(&cache_lock);
416. }
417. void do_item_stats_totals(ADD_STAT add_stats, void *c) {
418.     itemstats_t totals;
419.     memset(&totals, 0, sizeof(itemstats_t));
420.     int i;
421.     for (i = 0; i < LARGEST_ID; i++) {
422.         totals.expired_unfetched += itemstats[i].expired_unfetched;
423.         totals.evicted_unfetched += itemstats[i].evicted_unfetched;
424.         totals.evicted += itemstats[i].evicted;
425.         totals.reclaimed += itemstats[i].reclaimed;
426.         totals.crawler_reclaimed += itemstats[i].crawler_reclaimed;
427.     }
428.     APPEND_STAT("expired_unfetched", "%llu",
429.                 (unsigned long long)totals.expired_unfetched);
430.     APPEND_STAT("evicted_unfetched", "%llu",
431.                 (unsigned long long)totals.evicted_unfetched);
432.     APPEND_STAT("evictions", "%llu",
433.                 (unsigned long long)totals.evicted);
434.     APPEND_STAT("reclaimed", "%llu",
435.                 (unsigned long long)totals.reclaimed);
436.     APPEND_STAT("crawler_reclaimed", "%llu",
437.                 (unsigned long long)totals.crawler_reclaimed);
438. }
439. void do_item_stats(ADD_STAT add_stats, void *c) {
440.     int i;
441.     for (i = 0; i < LARGEST_ID; i++) {
442.         if (tails[i] != NULL) {
443.             const char *fmt = "items:%d:%s";
444.             char key_str[STAT_KEY_LEN];
445.             char val_str[STAT_VAL_LEN];
446.             int klen = 0, vlen = 0;
447.             if (tails[i] == NULL) {
448.                 /* We removed all of the items in this slab class */
449.                 continue;
450.             }
451.             APPEND_NUM_FMT_STAT(fmt, i, "number", "%u", sizes[i]);
452.             APPEND_NUM_FMT_STAT(fmt, i, "age", "%u", current_time - tails[i]->time);
453.             APPEND_NUM_FMT_STAT(fmt, i, "evicted",
454.                                 "%llu", (unsigned long long)itemstats[i].evicted);
455.             APPEND_NUM_FMT_STAT(fmt, i, "evicted_nonzero",
456.                                 "%llu", (unsigned long long)itemstats[i].evicted_nonzero);
457.             APPEND_NUM_FMT_STAT(fmt, i, "evicted_time",
458.                                 "%u", itemstats[i].evicted_time);
459.             APPEND_NUM_FMT_STAT(fmt, i, "outofmemory",
460.                                 "%llu", (unsigned long long)itemstats[i].outofmemory);
461.             APPEND_NUM_FMT_STAT(fmt, i, "tailrepairs",
462.                                 "%llu", (unsigned long long)itemstats[i].tailrepairs);
463.             APPEND_NUM_FMT_STAT(fmt, i, "reclaimed",
464.                                 "%llu", (unsigned long long)itemstats[i].reclaimed);
465.             APPEND_NUM_FMT_STAT(fmt, i, "expired_unfetched",
466.                                 "%llu", (unsigned long long)itemstats[i].expired_unfetched);
467.             APPEND_NUM_FMT_STAT(fmt, i, "evicted_unfetched",
468.                                 "%llu", (unsigned long long)itemstats[i].evicted_unfetched);
469.             APPEND_NUM_FMT_STAT(fmt, i, "crawler_reclaimed",
470.                                 "%llu", (unsigned long long)itemstats[i].crawler_reclaimed);
471.         }
472.     }
473.     /* getting here means both ascii and binary terminators fit */
474.     add_stats(NULL, 0, NULL, 0, c);
475. }
476. void do_item_stats_sizes(ADD_STAT add_stats, void *c) {
477.     /* max 1MB object, divided into 32 bytes size buckets */
478.     const int num_buckets = 32768;
479.     unsigned int *histogram = calloc(num_buckets, sizeof(int));
480.     if (histogram != NULL) {
481.         int i;
482.         /* build the histogram */
483.         for (i = 0; i < LARGEST_ID; i++) {
484.             item *iter = heads[i];
485.             while (iter) {
486.                 int ntotal = ITEM_ntotal(iter);
487.                 int bucket = ntotal / 32;
488.                 if ((ntotal % 32) != 0) bucket++;
489.                 if (bucket < num_buckets) histogram[bucket]++;
490.                 iter = iter->next;
491.             }
492.         }
493.         /* write the buffer */
494.         for (i = 0; i < num_buckets; i++) {
495.             if (histogram[i] != 0) {
496.                 char key[8];
497.                 snprintf(key, sizeof(key), "%d", i * 32);
498.                 APPEND_STAT(key, "%u", histogram[i]);
499.             }
500.         }
501.         free(histogram);
502.     }
503.     add_stats(NULL, 0, NULL, 0, c);
504. }
505.  
506. //读取item数据
507. item *do_item_get(const char *key, const size_t nkey, const uint32_t hv) {
508.     //mutex_lock(&cache_lock);
509.     item *it = assoc_find(key, nkey, hv);
510.     if (it != NULL) {
511.         refcount_incr(&it->refcount);
512.         if (slab_rebalance_signal &&
513.             ((void *)it >= slab_rebal.slab_start && (void *)it < slab_rebal.slab_end)) {
514.             do_item_unlink_nolock(it, hv);
515.             do_item_remove(it);
516.             it = NULL;
517.         }
518.     }
519.     //mutex_unlock(&cache_lock);
520.     int was_found = 0;
521.     if (settings.verbose > 2) {
522.         int ii;
523.         if (it == NULL) {
524.             fprintf(stderr, "> NOT FOUND ");
525.         } else {
526.             fprintf(stderr, "> FOUND KEY ");
527.             was_found++;
528.         }
529.         for (ii = 0; ii < nkey; ++ii) {
530.             fprintf(stderr, "%c", key[ii]);
531.         }
532.     }
533.     if (it != NULL) {
534.         if (settings.oldest_live != 0 && settings.oldest_live <= current_time &&
535.             it->time <= settings.oldest_live) {
536.             do_item_unlink(it, hv);
537.             do_item_remove(it);
538.             it = NULL;
539.             if (was_found) {
540.                 fprintf(stderr, " -nuked by flush");
541.             }
542.         } else if (it->exptime != 0 && it->exptime <= current_time) {
543.             do_item_unlink(it, hv);
544.             do_item_remove(it);
545.             it = NULL;
546.             if (was_found) {
547.                 fprintf(stderr, " -nuked by expire");
548.             }
549.         } else {
550.             it->it_flags |= ITEM_FETCHED;
551.             DEBUG_REFCNT(it, '+');
552.         }
553.     }
554.     if (settings.verbose > 2)
555.         fprintf(stderr, " ");
556.     return it;
557. }
558. item *do_item_touch(const char *key, size_t nkey, uint32_t exptime,
559.                     const uint32_t hv) {
560.     item *it = do_item_get(key, nkey, hv);
561.     if (it != NULL) {
562.         it->exptime = exptime;
563.     }
564.     return it;
565. }
566. /* expires items that are more recent than the oldest_live setting. */
567. void do_item_flush_expired(void) {
568.     int i;
569.     item *iter, *next;
570.     if (settings.oldest_live == 0)
571.         return;
572.     for (i = 0; i < LARGEST_ID; i++) {
573.         for (iter = heads[i]; iter != NULL; iter = next) {
574.             /* iter->time of 0 are magic objects. */
575.             if (iter->time != 0 && iter->time >= settings.oldest_live) {
576.                 next = iter->next;
577.                 if ((iter->it_flags & ITEM_SLABBED) == 0) {
578.                     do_item_unlink_nolock(iter, hash(ITEM_key(iter), iter->nkey));
579.                 }
580.             } else {
581.                 /* We've hit the first old item. Continue to the next queue. */
582.                 break;
583.             }
584.         }
585.     }
586. }
587. static void crawler_link_q(item *it) { /* item is the new tail */
588.     item **head, **tail;
589.     assert(it->slabs_clsid < LARGEST_ID);
590.     assert(it->it_flags == 1);
591.     assert(it->nbytes == 0);
592.     head = &heads[it->slabs_clsid];
593.     tail = &tails[it->slabs_clsid];
594.     assert(*tail != 0);
595.     assert(it != *tail);
596.     assert((*head && *tail) || (*head == 0 && *tail == 0));
597.     it->prev = *tail;
598.     it->next = 0;
599.     if (it->prev) {
600.         assert(it->prev->next == 0);
601.         it->prev->next = it;
602.     }
603.     *tail = it;
604.     if (*head == 0) *head = it;
605.     return;
606. }
607. static void crawler_unlink_q(item *it) {
608.     item **head, **tail;
609.     assert(it->slabs_clsid < LARGEST_ID);
610.     head = &heads[it->slabs_clsid];
611.     tail = &tails[it->slabs_clsid];
612.     if (*head == it) {
613.         assert(it->prev == 0);
614.         *head = it->next;
615.     }
616.     if (*tail == it) {
617.         assert(it->next == 0);
618.         *tail = it->prev;
619.     }
620.     assert(it->next != it);
621.     assert(it->prev != it);
622.     if (it->next) it->next->prev = it->prev;
623.     if (it->prev) it->prev->next = it->next;
624.     return;
625. }
626.  
627. static item *crawler_crawl_q(item *it) {
628.     item **head, **tail;
629.     assert(it->it_flags == 1);
630.     assert(it->nbytes == 0);
631.     assert(it->slabs_clsid < LARGEST_ID);
632.     head = &heads[it->slabs_clsid];
633.     tail = &tails[it->slabs_clsid];
634.     /* We've hit the head, pop off */
635.     if (it->prev == 0) {
636.         assert(*head == it);
637.         if (it->next) {
638.             *head = it->next;
639.             assert(it->next->prev == it);
640.             it->next->prev = 0;
641.         }
642.         return NULL; /* Done */
643.     }
644.     assert(it->prev != it);
645.     if (it->prev) {
646.         if (*head == it->prev) {
647.             *head = it;
648.         }
649.         if (*tail == it) {
650.             *tail = it->prev;
651.         }
652.         assert(it->next != it);
653.         if (it->next) {
654.             assert(it->prev->next == it);
655.             it->prev->next = it->next;
656.             it->next->prev = it->prev;
657.         } else {
658.             it->prev->next = 0;
659.         }
660.         it->next = it->prev;
661.         it->prev = it->next->prev;
662.         it->next->prev = it;
663.         if (it->prev) {
664.             it->prev->next = it;
665.         }
666.     }
667.     assert(it->next != it);
668.     assert(it->prev != it);
669.     return it->next; /* success */
670. }
671. /* I pulled this out to make the main thread clearer, but it reaches into the
672.  * main thread's values too much. Should rethink again.
673.  上面这句注释作者是说，他把用爬虫处理过期的item的工作放到另一个专门的线程里去做
674.  是为了让主线程干净一点，但是这线程的工作涉及到太多主线程的东西了，得重新想想..
675.  这个函数的作用是“评估”一下这个item是否应该free掉。其实主要就是看下有没有过期啦~
676.  当然用户设置的settings.oldest_live参数也加入到考虑中
677.  */
678. static void item_crawler_evaluate(item *search, uint32_t hv, int i) {
679.     rel_time_t oldest_live = settings.oldest_live;
680.     if ((search->exptime != 0 && search->exptime < current_time)
681.         || (search->time <= oldest_live && oldest_live <= current_time)) {
682.         itemstats[i].crawler_reclaimed++;
683.         if (settings.verbose > 1) {
684.             int ii;
685.             char *key = ITEM_key(search);
686.             fprintf(stderr, "LRU crawler found an expired item (flags: %d, slab: %d): ",
687.                 search->it_flags, search->slabs_clsid);
688.             for (ii = 0; ii < search->nkey; ++ii) {
689.                 fprintf(stderr, "%c", key[ii]);
690.             }
691.             fprintf(stderr, " ");
692.         }
693.         if ((search->it_flags & ITEM_FETCHED) == 0) {
694.             itemstats[i].expired_unfetched++;
695.         }
696.         do_item_unlink_nolock(search, hv);
697.         do_item_remove(search);
698.         assert(search->slabs_clsid == 0);
699.     } else {
700.         refcount_decr(&search->refcount);
701.     }
702. }
703. /**
704. item爬虫线程入口，负责从lru链表中把过期的item free掉
705. */
706. static void *item_crawler_thread(void *arg) {
707.     int i;
708.     pthread_mutex_lock(&lru_crawler_lock);
709.     if (settings.verbose > 2)
710.         fprintf(stderr, "Starting LRU crawler background thread ");
711.     while (do_run_lru_crawler_thread) {
712.     pthread_cond_wait(&lru_crawler_cond, &lru_crawler_lock);
713.     while (crawler_count) {
714.         item *search = NULL;
715.         void *hold_lock = NULL;
716.         for (i = 0; i < LARGEST_ID; i++) {
717.             if (crawlers[i].it_flags != 1) {
718.                 continue;
719.             }
720.             pthread_mutex_lock(&cache_lock);
721.             search = crawler_crawl_q((item *)&crawlers[i]);
722.             if (search == NULL ||
723.                 (crawlers[i].remaining && --crawlers[i].remaining < 1)) {
724.                 if (settings.verbose > 2)
725.                     fprintf(stderr, "Nothing left to crawl for %d ", i);
726.                 crawlers[i].it_flags = 0;
727.                 crawler_count--;
728.                 crawler_unlink_q((item *)&crawlers[i]);
729.                 pthread_mutex_unlock(&cache_lock);
730.                 continue;
731.             }
732.             uint32_t hv = hash(ITEM_key(search), search->nkey);
733.             /* Attempt to hash item lock the "search" item. If locked, no
734.              * other callers can incr the refcount
735.              */
736.             if ((hold_lock = item_trylock(hv)) == NULL) {
737.                 pthread_mutex_unlock(&cache_lock);
738.                 continue;
739.             }
740.             /* Now see if the item is refcount locked */
741.             if (refcount_incr(&search->refcount) != 2) {
742.                 refcount_decr(&search->refcount);
743.                 if (hold_lock)
744.                     item_trylock_unlock(hold_lock);
745.                 pthread_mutex_unlock(&cache_lock);
746.                 continue;
747.             }
748.             item_crawler_evaluate(search, hv, i);
749.             if (hold_lock)
750.                 item_trylock_unlock(hold_lock);
751.             pthread_mutex_unlock(&cache_lock);
752.             if (settings.lru_crawler_sleep)
753.                 usleep(settings.lru_crawler_sleep);
754.         }
755.     }
756.     if (settings.verbose > 2)
757.         fprintf(stderr, "LRU crawler thread sleeping ");
758.     STATS_LOCK();
759.     stats.lru_crawler_running = false;
760.     STATS_UNLOCK();
761.     }
762.     pthread_mutex_unlock(&lru_crawler_lock);
763.     if (settings.verbose > 2)
764.         fprintf(stderr, "LRU crawler thread stopping ");
765.     return NULL;
766. }
767. static pthread_t item_crawler_tid;
768. //停止item爬虫线程
769. int stop_item_crawler_thread(void) {
770.     int ret;
771.     pthread_mutex_lock(&lru_crawler_lock);
772.     do_run_lru_crawler_thread = 0;
773.     pthread_cond_signal(&lru_crawler_cond);
774.     pthread_mutex_unlock(&lru_crawler_lock);
775.     if ((ret = pthread_join(item_crawler_tid, NULL)) != 0) {
776.         fprintf(stderr, "Failed to stop LRU crawler thread: %s ", strerror(ret));
777.         return -1;
778.     }
779.     settings.lru_crawler = false;
780.     return 0;
781. }
782. /**
783. 启动item 爬虫线程
784. */
785. int start_item_crawler_thread(void) {
786.     int ret;
787.     if (settings.lru_crawler)
788.         return -1;
789.     pthread_mutex_lock(&lru_crawler_lock);
790.     do_run_lru_crawler_thread = 1;
791.     settings.lru_crawler = true;
792.     if ((ret = pthread_create(&item_crawler_tid, NULL,
793.         item_crawler_thread, NULL)) != 0) {
794.         fprintf(stderr, "Can't create LRU crawler thread: %s ",
795.             strerror(ret));
796.         pthread_mutex_unlock(&lru_crawler_lock);
797.         return -1;
798.     }
799.     pthread_mutex_unlock(&lru_crawler_lock);
800.     return 0;
801. }
802. enum crawler_result_type lru_crawler_crawl(char *slabs) {
803.     char *b = NULL;
804.     uint32_t sid = 0;
805.     uint8_t tocrawl[POWER_LARGEST];
806.     if (pthread_mutex_trylock(&lru_crawler_lock) != 0) {
807.         return CRAWLER_RUNNING;
808.     }
809.     pthread_mutex_lock(&cache_lock);
810.     if (strcmp(slabs, "all") == 0) {
811.         for (sid = 0; sid < LARGEST_ID; sid++) {
812.             tocrawl[sid] = 1;
813.         }
814.     } else {
815.         for (char *p = strtok_r(slabs, ",", &b);
816.              p != NULL;
817.              p = strtok_r(NULL, ",", &b)) {
818.             if (!safe_strtoul(p, &sid) || sid < POWER_SMALLEST
819.                     || sid > POWER_LARGEST) {
820.                 pthread_mutex_unlock(&cache_lock);
821.                 pthread_mutex_unlock(&lru_crawler_lock);
822.                 return CRAWLER_BADCLASS;
823.             }
824.             tocrawl[sid] = 1;
825.         }
826.     }
827.     for (sid = 0; sid < LARGEST_ID; sid++) {
828.         if (tocrawl[sid] != 0 && tails[sid] != NULL) {
829.             if (settings.verbose > 2)
830.                 fprintf(stderr, "Kicking LRU crawler off for slab %d ", sid);
831.             crawlers[sid].nbytes = 0;
832.             crawlers[sid].nkey = 0;
833.             crawlers[sid].it_flags = 1; /* For a crawler, this means enabled. */
834.             crawlers[sid].next = 0;
835.             crawlers[sid].prev = 0;
836.             crawlers[sid].time = 0;
837.             crawlers[sid].remaining = settings.lru_crawler_tocrawl;
838.             crawlers[sid].slabs_clsid = sid;
839.             crawler_link_q((item *)&crawlers[sid]);
840.             crawler_count++;
841.         }
842.     }
843.     pthread_mutex_unlock(&cache_lock);
844.     pthread_cond_signal(&lru_crawler_cond);
845.     STATS_LOCK();
846.     stats.lru_crawler_running = true;
847.     STATS_UNLOCK();
848.     pthread_mutex_unlock(&lru_crawler_lock);
849.     return CRAWLER_OK;
850. }
851. //初始化lru item爬虫线程
852. int init_lru_crawler(void) {
853.     if (lru_crawler_initialized == 0) {
854.         if (pthread_cond_init(&lru_crawler_cond, NULL) != 0) {
855.             fprintf(stderr, "Can't initialize lru crawler condition ");
856.             return -1;
857.         }
858.         pthread_mutex_init(&lru_crawler_lock, NULL);
859.         lru_crawler_initialized = 1;
860.     }
861.     return 0;
862. }