红黑树的操作：
1 /* 在rb中插入一个节点，成功返回NULL，否则返回error 2 * Insert a new node into the rbtree. This returns NULL on success, or the 3 * existing node on error. 4 */ 5 #define rb_insert(root, new, member, compar) 6 ({ 7 struct rb_node **__n = &(root)->rb_node, *__parent = NULL; 8 typeof(new) __old = NULL, __data; 9 10 while (*__n) { 11 __data = rb_entry(*__n, typeof(*new), member); 12 int __cmp = compar(new, __data); 13 14 __parent = *__n; 15 if (__cmp < 0) 16 __n = &((*__n)->rb_left); 17 else if (__cmp > 0) 18 __n = &((*__n)->rb_right); 19 else { 20 __old = __data; 21 break; 22 } 23 } 24 25 if (__old == NULL) { 26 /* Add new node and rebalance tree. */ 27 rb_link_node(&((new)->member), __parent, __n); 28 rb_insert_color(&((new)->member), root); 29 } 30 31 __old; 32 })
在rbtree中查找一个节点：
1 /* 2 * Search for a value in the rbtree. When the key is not found in the rbtree, 3 * this returns the next greater node. Note, if key > greatest node, we'll 4 * return first node. 5 * 6 * For an empty tree, we return NULL. 7 */ 8 #define rb_nsearch(root, key, member, compar) 9 ({ 10 struct rb_node *__n = (root)->rb_node; 11 typeof(key) __ret = NULL, __data; 12 13 while (__n) { 14 __data = rb_entry(__n, typeof(*key), member); 15 int __cmp = compar(key, __data); 16 17 if (__cmp < 0) { 18 __ret = __data; 19 __n = __n->rb_left; 20 } else if (__cmp > 0) 21 __n = __n->rb_right; 22 else { 23 __ret = __data; 24 break; 25 } 26 } 27 if (!__ret && !RB_EMPTY_ROOT(root)) 28 __ret = rb_entry(rb_first(root), typeof(*key), member); 29 __ret; 30 })
创建vdisks：一个物理disk的记录就是一颗红黑子树，如果删除该disk，那么对应的是删除整棵子树
1 static void create_vdisks(const struct disk *disk) 2 { 3 //计算disk路径的hash值 4 uint64_t hval = sd_hash(disk->path, strlen(disk->path)); 5 const struct sd_node *n = &sys->this_node; 6 uint64_t node_hval; 7 int nr; 8 // 判断当前使用的模式：Disk mode for cluster 9 if (is_cluster_diskmode(&sys->cinfo)) { 10 // node的hash值 11 node_hval = sd_hash(&n->nid, offsetof(typeof(n->nid), io_addr)); 13 hval = fnv_64a_64(node_hval, hval); 14 // nr即将disk按照4G大小划分后的个数 15 nr = DIV_ROUND_UP(disk->space, WEIGHT_MIN); 16 if (0 == n->nid.port) 17 return; 18 } else 19 // 按照16M空间大小划分磁盘空间后的个数 20 nr = vdisk_number(disk); 21 // 上述主要是获取到vdisk的个数，接下来将vdisk信息存储在rbtree中 22 for (int i = 0; i < nr; i++) { 23 struct vdisk *v = xmalloc(sizeof(*v)); 25 hval = sd_hash_next(hval); 26 v->hash = hval; 27 v->disk = disk; 28 if (unlikely(vdisk_insert(v))) // 在rbtree中插入vdisk，树的根在md.vroot这样的全局变量中 29 panic("vdisk hash collison"); 30 } 31 }
添加一个物理磁盘：
1 /* We don't need lock at init stage */
2 bool md_add_disk(const char *path, bool purge)
3 {
4 struct disk *new;
5 // 是否已经存在
6 if (path_to_disk(path)) {
7 sd_err("duplicate path %s", path);
8 return false;
9 }
10 // 创建相应的文件夹目录
11 if (xmkdir(path, sd_def_dmode) < 0) {
12 sd_err("can't mkdir for %s, %m", path);
13 return false;
14 }
15
16 new = xmalloc(sizeof(*new));
17 pstrcpy(new->path, PATH_MAX, path);
18 trim_last_slash(new->path);
19 new->space = init_path_space(new->path, purge);
20 if (!new->space) {
21 free(new);
22 return false;
23 }
24 // 创建相应的vdisks
25 create_vdisks(new);
26 rb_insert(&md.root, new, rb, disk_cmp);
27 md.space += new->space;
28 md.nr_disks++;
29
30 sd_info("%s, vdisk nr %d, total disk %d", new->path, vdisk_number(new),md.nr_disks);
31 return true;
32 }
与之相反的是删除vdisk
1 static inline void vdisk_free(struct vdisk *v) 2 { 3 // 删除树的结构 4 rb_erase(&v->rb, &md.vroot); 5 free(v); 6 } 7 8 static void remove_vdisks(const struct disk *disk) 9 { 10 uint64_t hval = sd_hash(disk->path, strlen(disk->path)); 11 const struct sd_node *n = &sys->this_node; 12 uint64_t node_hval; 13 int nr; 14 15 if (is_cluster_diskmode(&sys->cinfo)) { 16 node_hval = sd_hash(&n->nid, offsetof(typeof(n->nid), io_addr)); 17 hval = fnv_64a_64(node_hval, hval); 18 nr = DIV_ROUND_UP(disk->space, WEIGHT_MIN); 19 } else 20 nr = vdisk_number(disk); 21 22 for (int i = 0; i < nr; i++) { 23 struct vdisk *v; 24 25 hval = sd_hash_next(hval); 26 v = hval_to_vdisk(hval); 27 sd_assert(v->hash == hval); 28 // 逐个释放vdisk 29 vdisk_free(v); 30 } 31 }
删除物理disk操作：
1 static void remove_vdisks(const struct disk *disk) 2 { 3 uint64_t hval = sd_hash(disk->path, strlen(disk->path)); 4 const struct sd_node *n = &sys->this_node; 5 uint64_t node_hval; 6 int nr; 7 8 if (is_cluster_diskmode(&sys->cinfo)) { 9 node_hval = sd_hash(&n->nid, offsetof(typeof(n->nid), io_addr)); 10 hval = fnv_64a_64(node_hval, hval); 11 nr = DIV_ROUND_UP(disk->space, WEIGHT_MIN); 12 } else 13 nr = vdisk_number(disk); 14 15 for (int i = 0; i < nr; i++) { 16 struct vdisk *v; 17 18 hval = sd_hash_next(hval); 19 v = hval_to_vdisk(hval); 20 sd_assert(v->hash == hval); 21 22 vdisk_free(v); 23 } 24 } 25 26 27 static inline void md_remove_disk(struct disk *disk) 28 { 29 sd_info("%s from multi-disk array", disk->path); 30 rb_erase(&disk->rb, &md.root); 31 md.nr_disks--; 32 remove_vdisks(disk); 33 free(disk); 34 } 35 36 static inline void md_del_disk(const char *path) 37 { 38 struct disk *disk = path_to_disk(path); 39 40 if (!disk) { 41 sd_err("invalid path %s", path); 42 return; 43 } 44 md_remove_disk(disk); 45 }
更新nodes disks，这里仅在配置为disk_vnodes模式才会定义：
1 #ifdef HAVE_DISKVNODES 2 void update_node_disks(void) 3 { 4 const struct disk *disk; 5 int i = 0; 6 bool rb_empty = false; 7 8 if (!sys) 9 return; 10 11 memset(sys->this_node.disks, 0, sizeof(struct disk_info) * DISK_MAX); 12 sd_read_lock(&md.lock); 13 rb_for_each_entry(disk, &md.root, rb) { 14 sys->this_node.disks[i].disk_id = 15 sd_hash(disk->path, strlen(disk->path)); 16 sys->this_node.disks[i].disk_space = disk->space; 17 i++; 18 } 19 sd_rw_unlock(&md.lock); 20 21 if (RB_EMPTY_ROOT(&md.vroot)) 22 rb_empty = true; 23 sd_write_lock(&md.lock); 24 rb_for_each_entry(disk, &md.root, rb) { 25 if (!rb_empty) 26 remove_vdisks(disk); 27 create_vdisks(disk); 28 } 29 sd_rw_unlock(&md.lock); 30 } 31 #else 32 void update_node_disks(void) 33 { 34 } 35 #endif
本地磁盘插拔操作：
1 static int do_plug_unplug(char *disks, bool plug) 2 { 3 const char *path; 4 int old_nr, new_nr, ret = SD_RES_UNKNOWN; 5 6 sd_write_lock(&md.lock); 7 old_nr = md.nr_disks; 8 path = strtok(disks, ","); 9 do { 10 if (plug) { 11 if (!md_add_disk(path, true)) 12 sd_err("failed to add %s", path); 13 } else { 14 md_del_disk(path); 15 } 16 } while ((path = strtok(NULL, ","))); 17 new_nr = md.nr_disks; 18 19 /* If no disks change, bail out */ 20 if (old_nr == new_nr) 21 goto out; 22 23 ret = SD_RES_SUCCESS; 24 out: 25 sd_rw_unlock(&md.lock); 26 27 if (ret == SD_RES_SUCCESS) { 28 if (new_nr > 0) { 29 update_node_disks(); 30 kick_recover(); 31 } else { 32 sd_warn("no disks plugged, going down"); 33 leave_cluster(); 34 sys->cinfo.status = SD_STATUS_KILLED; 35 } 36 } 37 38 return ret; 39 } 40 41 int md_plug_disks(char *disks) 42 { 43 return do_plug_unplug(disks, true); 44 } 45 46 int md_unplug_disks(char *disks) 47 { 48 return do_plug_unplug(disks, false); 49 }
oid到vdisk的映射：
1 /* If v1_hash < hval <= v2_hash, then oid is resident in v2， 在rbtree中寻找其位置*/ 2 static struct vdisk *hval_to_vdisk(uint64_t hval) 3 { 4 struct vdisk dummy = { .hash = hval }; 5 6 return rb_nsearch(&md.vroot, &dummy, rb, vdisk_cmp); 7 } 8 /* 将oid的hash值作为入参 */ 9 static struct vdisk *oid_to_vdisk(uint64_t oid) 10 { 11 return hval_to_vdisk(sd_hash_oid(oid)); 12 }