和zipmap作为dict数据结构的备选一样,ziplist同样追求牺牲性能的情况下节省内存,毕竟redis被看做是一个in-memory的nosql工具。
zipmap的源码中,印象最深的两个细节是:
1. 作者为了节省内存,如果存入的value是一个能转化成int的数字串,存储的时候,ziplist会将其转换成int类型存储
2. 同样为了节省内存,在存储长度的字段都是用1or5的存储方式,当被存储字段长度小于254的时候,使用一个字节,大于等于254的时候再使用额外四个字节去存储长度【吐槽一下,代码中最麻烦的那30%的代码都和这个细节有关~~~~~~如果直接用4个字节不好吗,亲?】
添加了简单注释的代码如下
Ziplist.h
1 #define ZIPLIST_HEAD 0 2 #define ZIPLIST_TAIL 1 3 4 unsigned char *ziplistNew(void); 5 unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where); 6 unsigned char *ziplistIndex(unsigned char *zl, int index); 7 unsigned char *ziplistNext(unsigned char *zl, unsigned char *p); 8 unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p); 9 unsigned int ziplistGet(unsigned char *p, unsigned char **sval, unsigned int *slen, long long *lval); 10 unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen); 11 unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p); 12 unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num); 13 unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int slen); 14 unsigned int ziplistLen(unsigned char *zl); 15 unsigned int ziplistSize(unsigned char *zl);
Ziplist.c
1 /* The ziplist is a specially encoded dually linked list that is designed 2 * to be very memory efficient. It stores both strings and integer values, 3 * where integers are encoded as actual integers instead of a series of 4 * characters. It allows push and pop operations on either side of the list 5 * in O(1) time. However, because every operation requires a reallocation of 6 * the memory used by the ziplist, the actual complexity is related to the 7 * amount of memory used by the ziplist. 8 * 9 * ---------------------------------------------------------------------------- 10 * 11 * ZIPLIST OVERALL LAYOUT: 12 * The general layout of the ziplist is as follows: 13 * <zlbytes><zltail><zllen><entry><entry><zlend> 14 * 15 * <zlbytes> is an unsigned integer to hold the number of bytes that the 16 * ziplist occupies. This value needs to be stored to be able to resize the 17 * entire structure without the need to traverse it first. 18 * 19 * <zltail> is the offset to the last entry in the list. This allows a pop 20 * operation on the far side of the list without the need for full traversal. 21 * 22 * <zllen> is the number of entries.When this value is larger than 2**16-2, 23 * we need to traverse the entire list to know how many items it holds. 24 * 25 * <zlend> is a single byte special value, equal to 255, which indicates the 26 * end of the list. 27 * 28 * ZIPLIST ENTRIES: 29 * Every entry in the ziplist is prefixed by a header that contains two pieces 30 * of information. First, the length of the previous entry is stored to be 31 * able to traverse the list from back to front. Second, the encoding with an 32 * optional string length of the entry itself is stored. 33 * 34 * The length of the previous entry is encoded in the following way: 35 * If this length is smaller than 254 bytes, it will only consume a single 36 * byte that takes the length as value. When the length is greater than or 37 * equal to 254, it will consume 5 bytes. The first byte is set to 254 to 38 * indicate a larger value is following. The remaining 4 bytes take the 39 * length of the previous entry as value. 40 * 41 * The other header field of the entry itself depends on the contents of the 42 * entry. When the entry is a string, the first 2 bits of this header will hold 43 * the type of encoding used to store the length of the string, followed by the 44 * actual length of the string. When the entry is an integer the first 2 bits 45 * are both set to 1. The following 2 bits are used to specify what kind of 46 * integer will be stored after this header. An overview of the different 47 * types and encodings is as follows: 48 * 49 * |00pppppp| - 1 byte 50 * String value with length less than or equal to 63 bytes (6 bits). 51 * |01pppppp|qqqqqqqq| - 2 bytes 52 * String value with length less than or equal to 16383 bytes (14 bits). 53 * |10______|qqqqqqqq|rrrrrrrr|ssssssss|tttttttt| - 5 bytes 54 * String value with length greater than or equal to 16384 bytes. 55 * |1100____| - 1 byte 56 * Integer encoded as int16_t (2 bytes). 57 * |1101____| - 1 byte 58 * Integer encoded as int32_t (4 bytes). 59 * |1110____| - 1 byte 60 * Integer encoded as int64_t (8 bytes). 61 */ 62 63 #include <stdio.h> 64 #include <stdlib.h> 65 #include <string.h> 66 #include <stdint.h> 67 #include <assert.h> 68 #include <limits.h> 69 #include "zmalloc.h" 70 #include "ziplist.h" 71 72 int ll2string(char *s, size_t len, long long value); 73 74 #define ZIP_END 255 75 #define ZIP_BIGLEN 254 76 77 //<zlbytes><zltail><zllen><entry><entry><zlend> 4 + 4 + 2 + sizeof(entry)* + 4 78 /* Different encoding/length possibilities */ 79 80 //<entry> = <len_of_prev_entry> + <info_of_current_entry> 81 #define ZIP_STR_06B (0 << 6) // 0000 0000 82 #define ZIP_STR_14B (1 << 6) // 0100 0000 83 #define ZIP_STR_32B (2 << 6) // 1000 0000 84 #define ZIP_INT_16B (0xc0 | 0<<4) //1100 0000 85 #define ZIP_INT_32B (0xc0 | 1<<4) // 1101 0000 86 #define ZIP_INT_64B (0xc0 | 2<<4) // 1110 0000 87 88 /* Macro's to determine type */ 89 #define ZIP_IS_STR(enc) (((enc) & 0xc0) < 0xc0) //根据entry第二个字段的第一个字节来判断entry中数据类型 90 #define ZIP_IS_INT(enc) (!ZIP_IS_STR(enc) && ((enc) & 0x30) < 0x30) 91 92 /* Utility macros */ 93 #define ZIPLIST_BYTES(zl) (*((uint32_t*)(zl))) //得到zl占用的内存大小 94 #define ZIPLIST_TAIL_OFFSET(zl) (*((uint32_t*)((zl)+sizeof(uint32_t))))//得到最后一个entry的地址【相对地址offset】 95 #define ZIPLIST_LENGTH(zl) (*((uint16_t*)((zl)+sizeof(uint32_t)*2))) //得到entry的数量 96 #define ZIPLIST_HEADER_SIZE (sizeof(uint32_t)*2+sizeof(uint16_t)) //得到zl数据<entry>前字段的总大小 97 #define ZIPLIST_ENTRY_HEAD(zl) ((zl)+ZIPLIST_HEADER_SIZE) //得到zl第一个entry的地址 98 #define ZIPLIST_ENTRY_TAIL(zl) ((zl)+ZIPLIST_TAIL_OFFSET(zl))//得到zl最后一个元素地址 99 #define ZIPLIST_ENTRY_END(zl) ((zl)+ZIPLIST_BYTES(zl)-1) //得到zl最后一个字节【255】的指针 100 101 /* We know a positive increment can only be 1 because entries can only be 102 * pushed one at a time. */ 103 //如果zllen字段还有效,即保存的数量没有超过16个字节能容纳的最大整数,则计数 104 #define ZIPLIST_INCR_LENGTH(zl,incr) { \ 105 if (ZIPLIST_LENGTH(zl) < UINT16_MAX) ZIPLIST_LENGTH(zl)+=incr; } 106 107 typedef struct zlentry { 108 unsigned int prevrawlensize, prevrawlen; 109 unsigned int lensize, len; 110 unsigned int headersize; 111 unsigned char encoding; 112 unsigned char *p; 113 } zlentry; 114 115 //返回encoding类型,分别是字符串value的encoding和int类型的encoding,各三种 116 //ZIP_STR_06B (0 << 6) 117 //ZIP_STR_14B (1 << 6) 118 //ZIP_STR_32B (2 << 6) 119 //ZIP_INT_16B (0xc0 | 0<<4) 120 //ZIP_INT_32B (0xc0 | 1<<4) 121 //ZIP_INT_64B (0xc0 | 2<<4) 122 /* Return the encoding pointer to by 'p'. */ 123 static unsigned int zipEntryEncoding(unsigned char *p) { 124 /* String encoding: 2 MSBs */ 125 unsigned char b = p[0] & 0xc0; 126 if (b < 0xc0) { //如果是字符串 127 return b; 128 } else { 129 /* Integer encoding: 4 MSBs */ 130 return p[0] & 0xf0; 131 } 132 assert(NULL); 133 return 0; 134 } 135 136 //如果entry的value是int类型,根据encoding方式字段得到int数据类型 137 /* Return bytes needed to store integer encoded by 'encoding' */ 138 static unsigned int zipIntSize(unsigned char encoding) { 139 switch(encoding) { 140 case ZIP_INT_16B: return sizeof(int16_t); 141 case ZIP_INT_32B: return sizeof(int32_t); 142 case ZIP_INT_64B: return sizeof(int64_t); 143 } 144 assert(NULL); 145 return 0; 146 } 147 148 //返回entry的字符串类型的value字段的长度,如果提供了lensize, 149 //则将其赋值为entry的header的所占字节数量 150 /* Decode the encoded length pointed by 'p'. If a pointer to 'lensize' is 151 * provided, it is set to the number of bytes required to encode the length. */ 152 static unsigned int zipDecodeLength(unsigned char *p, unsigned int *lensize) { 153 unsigned char encoding = zipEntryEncoding(p); 154 unsigned int len = 0; 155 156 if (ZIP_IS_STR(encoding)) { 157 switch(encoding) { 158 case ZIP_STR_06B: 159 len = p[0] & 0x3f; 160 if (lensize) *lensize = 1; 161 break; 162 case ZIP_STR_14B: 163 len = ((p[0] & 0x3f) << 8) | p[1]; 164 if (lensize) *lensize = 2; 165 break; 166 case ZIP_STR_32B: 167 len = (p[1] << 24) | (p[2] << 16) | (p[3] << 8) | p[4]; 168 if (lensize) *lensize = 5; 169 break; 170 default: 171 assert(NULL); 172 } 173 } else { 174 len = zipIntSize(encoding); 175 if (lensize) *lensize = 1; 176 } 177 return len; 178 } 179 180 //根据存放数据的长度和encoding类型,为entryheader赋值,并且返回header所占字节数 181 /* Encode the length 'l' writing it in 'p'. If p is NULL it just returns 182 * the amount of bytes required to encode such a length. */ 183 static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, unsigned int rawlen) { 184 unsigned char len = 1, buf[5]; 185 186 if (ZIP_IS_STR(encoding)) { 187 /* Although encoding is given it may not be set for strings, 188 * so we determine it here using the raw length. */ 189 if (rawlen <= 0x3f) { 190 if (!p) return len; 191 buf[0] = ZIP_STR_06B | rawlen; 192 } else if (rawlen <= 0x3fff) { 193 len += 1; 194 if (!p) return len; 195 buf[0] = ZIP_STR_14B | ((rawlen >> 8) & 0x3f); 196 buf[1] = rawlen & 0xff; 197 } else { 198 len += 4; 199 if (!p) return len; 200 buf[0] = ZIP_STR_32B; 201 buf[1] = (rawlen >> 24) & 0xff; 202 buf[2] = (rawlen >> 16) & 0xff; 203 buf[3] = (rawlen >> 8) & 0xff; 204 buf[4] = rawlen & 0xff; 205 } 206 } else { 207 /* Implies integer encoding, so length is always 1. */ 208 if (!p) return len; 209 buf[0] = encoding; 210 } 211 212 /* Store this length at p */ 213 memcpy(p,buf,len); 214 return len; 215 } 216 217 218 //给出一个entry的头指针,计算其前一个entry的长度,并将其长度返回 219 //如果lensize不为NULL,返回这一字段所占字节数 220 /* Decode the length of the previous element stored at "p". */ 221 static unsigned int zipPrevDecodeLength(unsigned char *p, unsigned int *lensize) { 222 unsigned int len = *p; 223 if (len < ZIP_BIGLEN) { 224 if (lensize) *lensize = 1; 225 } else { 226 if (lensize) *lensize = 1+sizeof(len); 227 memcpy(&len,p+1,sizeof(len)); 228 } 229 return len; 230 } 231 232 233 234 //提供前一个entry的长度,得到current entry的第一字段长度,如果p不为NULL, 则将其长度encode后放入p的第一个字段 235 /* Encode the length of the previous entry and write it to "p". Return the 236 * number of bytes needed to encode this length if "p" is NULL. */ 237 static unsigned int zipPrevEncodeLength(unsigned char *p, unsigned int len) { 238 if (p == NULL) { 239 return (len < ZIP_BIGLEN) ? 1 : sizeof(len)+1; 240 } else { 241 if (len < ZIP_BIGLEN) { 242 p[0] = len; 243 return 1; 244 } else { 245 p[0] = ZIP_BIGLEN; 246 memcpy(p+1,&len,sizeof(len)); 247 return 1+sizeof(len); 248 } 249 } 250 } 251 252 253 //强制将上一个字段的长度以larger encoding的方式写入current entry的第一个字段 254 /* Encode the length of the previous entry and write it to "p". This only 255 * uses the larger encoding (required in __ziplistCascadeUpdate). */ 256 static void zipPrevEncodeLengthForceLarge(unsigned char *p, unsigned int len) { 257 if (p == NULL) return; 258 p[0] = ZIP_BIGLEN; 259 memcpy(p+1,&len,sizeof(len)); 260 } 261 262 //如果p指向的entry的长度变为len,则返回前一个entry中的第一个字段所需要的长度与改变前所需要的长度的差,单位是bytes 263 /* Return the difference in number of bytes needed to store the new length 264 * "len" on the entry pointed to by "p". */ 265 static int zipPrevLenByteDiff(unsigned char *p, unsigned int len) { 266 unsigned int prevlensize; 267 zipPrevDecodeLength(p,&prevlensize); 268 return zipPrevEncodeLength(NULL,len)-prevlensize; 269 } 270 271 272 273 //检查entry指针开头的entry的字符串数据是否可以被转成integer,如果可以返回1,否则返回0,并将拿到的数字赋值给*v, 将encoding字段赋值给 *encoding 274 /* Check if string pointed to by 'entry' can be encoded as an integer. 275 * Stores the integer value in 'v' and its encoding in 'encoding'. */ 276 static int zipTryEncoding(unsigned char *entry, unsigned int entrylen, long long *v, unsigned char *encoding) { 277 long long value; 278 char *eptr; 279 char buf[32]; 280 281 if (entrylen >= 32 || entrylen == 0) return 0; 282 if (entry[0] == '-' || (entry[0] >= '0' && entry[0] <= '9')) { 283 int slen; 284 285 /* Perform a back-and-forth conversion to make sure that 286 * the string turned into an integer is not losing any info. */ 287 memcpy(buf,entry,entrylen); 288 buf[entrylen] = '\0'; 289 value = strtoll(buf,&eptr,10); 290 if (eptr[0] != '\0') return 0; 291 slen = ll2string(buf,32,value); 292 if (entrylen != (unsigned)slen || memcmp(buf,entry,slen)) return 0; 293 294 /* Great, the string can be encoded. Check what's the smallest 295 * of our encoding types that can hold this value. */ 296 if (value >= INT16_MIN && value <= INT16_MAX) { 297 *encoding = ZIP_INT_16B; 298 } else if (value >= INT32_MIN && value <= INT32_MAX) { 299 *encoding = ZIP_INT_32B; 300 } else { 301 *encoding = ZIP_INT_64B; 302 } 303 *v = value; 304 return 1; 305 } 306 return 0; 307 } 308 309 //在p开始的value字段存放int类型的数据 310 /* Store integer 'value' at 'p', encoded as 'encoding' */ 311 static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encoding) { 312 int16_t i16; 313 int32_t i32; 314 int64_t i64; 315 if (encoding == ZIP_INT_16B) { 316 i16 = value; 317 memcpy(p,&i16,sizeof(i16)); 318 } else if (encoding == ZIP_INT_32B) { 319 i32 = value; 320 memcpy(p,&i32,sizeof(i32)); 321 } else if (encoding == ZIP_INT_64B) { 322 i64 = value; 323 memcpy(p,&i64,sizeof(i64)); 324 } else { 325 assert(NULL); 326 } 327 } 328 329 //从p开始的value中,根据encoding的方式,提取存放的int类型value 330 /* Read integer encoded as 'encoding' from 'p' */ 331 static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) { 332 int16_t i16; 333 int32_t i32; 334 int64_t i64, ret = 0; 335 if (encoding == ZIP_INT_16B) { 336 memcpy(&i16,p,sizeof(i16)); 337 ret = i16; 338 } else if (encoding == ZIP_INT_32B) { 339 memcpy(&i32,p,sizeof(i32)); 340 ret = i32; 341 } else if (encoding == ZIP_INT_64B) { 342 memcpy(&i64,p,sizeof(i64)); 343 ret = i64; 344 } else { 345 assert(NULL); 346 } 347 return ret; 348 } 349 350 /* 351 typedef struct zlentry { 352 unsigned int prevrawlensize[前一个entry的长度字段需要的byte数], prevrawlen【前一个entry数据长度】; 353 unsigned int lensize【本entry的encoding字段所需要的字节数】, len【本字段数据长度】; 354 unsigned int headersize【本etry的header占用的字节数】; 355 unsigned char encoding;【本字段的encoding】 356 unsigned char *p;【本entry头指针】 357 } zlentry; 358 */ 359 // 360 /* Return a struct with all information about an entry. */ 361 static zlentry zipEntry(unsigned char *p) { 362 zlentry e; 363 e.prevrawlen = zipPrevDecodeLength(p,&e.prevrawlensize); 364 e.len = zipDecodeLength(p+e.prevrawlensize,&e.lensize); 365 e.headersize = e.prevrawlensize+e.lensize; 366 e.encoding = zipEntryEncoding(p+e.prevrawlensize); 367 e.p = p; 368 return e; 369 } 370 371 //返回本字段所需要的所有字节数 372 /* Return the total number of bytes used by the entry at "p". */ 373 static unsigned int zipRawEntryLength(unsigned char *p) { 374 zlentry e = zipEntry(p); 375 return e.headersize + e.len; 376 } 377 378 //创建一个新的ziplist,空的,只有 379 //zlbytes,zloffset,zllen,[空entry],zlend 380 /* Create a new empty ziplist. */ 381 unsigned char *ziplistNew(void) { 382 unsigned int bytes = ZIPLIST_HEADER_SIZE+1; 383 unsigned char *zl = zmalloc(bytes); 384 ZIPLIST_BYTES(zl) = bytes; 385 ZIPLIST_TAIL_OFFSET(zl) = ZIPLIST_HEADER_SIZE; 386 ZIPLIST_LENGTH(zl) = 0; 387 zl[bytes-1] = ZIP_END; 388 return zl; 389 } 390 391 392 //为ziplist重新申请内存,修改zl相应字段 393 /* Resize the ziplist. */ 394 static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) { 395 zl = zrealloc(zl,len); 396 ZIPLIST_BYTES(zl) = len; 397 zl[len-1] = ZIP_END; 398 return zl; 399 } 400 401 402 //插入一个entry的时候,如果它下个的entry的prevlen字段的长度和插入的这个entry的 403 //len长度不同,并且需要由一个byte增加到5个byte,可能做这个修改不需要额外申请 404 //空间,因为插入本身就会realloc内存并将内存移位。可是reencoding这个字段本身又会造成 405 //整个entry长度的变化。这会可能造成下下个entry的prelen字段变化,并且循环下去 406 407 //这个问题可能会另一个方式发生,如果插入字段的后一个字段本来需要5个byte,结果 408 //插入新节点后,只需要1个byte便能保存 prelen 。 409 410 //第二种情况可以忽略,因为这个prelen字段可能在一些列操作中反复的变长变短。 411 //所以,这个字段可以允许比需要的要长些,因为大一点的prevlen字段表示整个ziplist 412 //holdoing了一些比较大的entries 413 414 415 /* When an entry is inserted, we need to set the prevlen field of the next 416 * entry to equal the length of the inserted entry. It can occur that this 417 * length cannot be encoded in 1 byte and the next entry needs to be grow 418 * a bit larger to hold the 5-byte encoded prevlen. This can be done for free, 419 * because this only happens when an entry is already being inserted (which 420 * causes a realloc and memmove). However, encoding the prevlen may require 421 * that this entry is grown as well. This effect may cascade throughout 422 * the ziplist when there are consecutive entries with a size close to 423 * ZIP_BIGLEN, so we need to check that the prevlen can be encoded in every 424 * consecutive entry. 425 * 426 * Note that this effect can also happen in reverse, where the bytes required 427 * to encode the prevlen field can shrink. This effect is deliberately ignored, 428 * because it can cause a "flapping" effect where a chain prevlen fields is 429 * first grown and then shrunk again after consecutive inserts. Rather, the 430 * field is allowed to stay larger than necessary, because a large prevlen 431 * field implies the ziplist is holding large entries anyway. 432 * 433 * The pointer "p" points to the first entry that does NOT need to be 434 * updated, i.e. consecutive fields MAY need an update. */ 435 static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p) { 436 unsigned int curlen = ZIPLIST_BYTES(zl), rawlen, rawlensize; 437 unsigned int offset, noffset, extra; 438 unsigned char *np; 439 zlentry cur, next; 440 441 while (p[0] != ZIP_END) { 442 cur = zipEntry(p); 443 rawlen = cur.headersize + cur.len; 444 rawlensize = zipPrevEncodeLength(NULL,rawlen); 445 446 /* Abort if there is no next entry. */ 447 if (p[rawlen] == ZIP_END) break; 448 next = zipEntry(p+rawlen); 449 450 /* Abort when "prevlen" has not changed. */ 451 if (next.prevrawlen == rawlen) break; 452 453 if (next.prevrawlensize < rawlensize) { 454 /* The "prevlen" field of "next" needs more bytes to hold 455 * the raw length of "cur". */ 456 offset = p-zl; 457 extra = rawlensize-next.prevrawlensize; 458 zl = ziplistResize(zl,curlen+extra); 459 p = zl+offset; 460 461 /* Current pointer and offset for next element. */ 462 np = p+rawlen; 463 noffset = np-zl; 464 465 /* Update tail offset when next element is not the tail element. */ 466 if ((zl+ZIPLIST_TAIL_OFFSET(zl)) != np) 467 ZIPLIST_TAIL_OFFSET(zl) += extra; 468 469 /* Move the tail to the back. */ 470 memmove(np+rawlensize, 471 np+next.prevrawlensize, 472 curlen-noffset-next.prevrawlensize-1); 473 zipPrevEncodeLength(np,rawlen); 474 475 /* Advance the cursor */ 476 p += rawlen; 477 curlen += extra; 478 } else { 479 if (next.prevrawlensize > rawlensize) { 480 /* This would result in shrinking, which we want to avoid. 481 * So, set "rawlen" in the available bytes. */ 482 zipPrevEncodeLengthForceLarge(p+rawlen,rawlen); 483 } else { 484 zipPrevEncodeLength(p+rawlen,rawlen); 485 } 486 487 /* Stop here, as the raw length of "next" has not changed. */ 488 break; 489 } 490 } 491 return zl; 492 } 493 494 //从p指定entry开始,删除连续的num个entry 495 /* Delete "num" entries, starting at "p". Returns pointer to the ziplist. */ 496 static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsigned int num) { 497 unsigned int i, totlen, deleted = 0; 498 int offset, nextdiff = 0; 499 zlentry first, tail; 500 501 //计算指定删除区域的最后一段的下一段的entry头指针 502 first = zipEntry(p); 503 for (i = 0; p[0] != ZIP_END && i < num; i++) { 504 p += zipRawEntryLength(p); 505 deleted++; 506 } 507 //需要删除的总字节数totlen 508 totlen = p-first.p; 509 if (totlen > 0) { 510 //如果删除的最后一段的下一段不是ZIP_END 511 if (p[0] != ZIP_END) { 512 /* Tricky: storing the prevlen in this entry might reduce or 513 * increase the number of bytes needed, compared to the current 514 * prevlen. Note that we can always store this length because 515 * it was previously stored by an entry that is being deleted. */ 516 //得到删除区域的下一段的prevlen和删除区域前一段的长度作比较,得到diff 517 nextdiff = zipPrevLenByteDiff(p,first.prevrawlen); 518 zipPrevEncodeLength(p-nextdiff,first.prevrawlen); 519 520 /* Update offset for tail */ 521 ZIPLIST_TAIL_OFFSET(zl) -= totlen; 522 523 /* When the tail contains more than one entry, we need to take 524 * "nextdiff" in account as well. Otherwise, a change in the 525 * size of prevlen doesn't have an effect on the *tail* offset. */ 526 tail = zipEntry(p); 527 if (p[tail.headersize+tail.len] != ZIP_END) 528 ZIPLIST_TAIL_OFFSET(zl) += nextdiff; 529 530 /* Move tail to the front of the ziplist */ 531 memmove(first.p,p-nextdiff,ZIPLIST_BYTES(zl)-(p-zl)-1+nextdiff); 532 } else { 533 /* The entire tail was deleted. No need to move memory. */ 534 ZIPLIST_TAIL_OFFSET(zl) = (first.p-zl)-first.prevrawlen; 535 } 536 537 /* Resize and update length */ 538 offset = first.p-zl; 539 zl = ziplistResize(zl, ZIPLIST_BYTES(zl)-totlen+nextdiff); 540 ZIPLIST_INCR_LENGTH(zl,-deleted); 541 p = zl+offset; 542 543 /* When nextdiff != 0, the raw length of the next entry has changed, so 544 * we need to cascade the update throughout the ziplist */ 545 if (nextdiff != 0) 546 zl = __ziplistCascadeUpdate(zl,p); 547 } 548 return zl; 549 } 550 551 //在位置p插入一个新entry, 552 /* Insert item at "p". */ 553 static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) { 554 unsigned int curlen = ZIPLIST_BYTES(zl), reqlen, prevlen = 0; 555 unsigned int offset, nextdiff = 0; 556 unsigned char encoding = 0; 557 long long value; 558 zlentry entry, tail; 559 560 /* Find out prevlen for the entry that is inserted. */ 561 //如果插入的位置不是最后一个,得到p所在的entry的prevlen字段的大小,准备赋给新entry的prevlen字段 562 if (p[0] != ZIP_END) { 563 entry = zipEntry(p); 564 prevlen = entry.prevrawlen; 565 //如果插入的位置是最后一个,即在zl的最后push进一个entry,得到tail元素的prevlen字段的值,准备赋给新entry的prevlen字段 566 } else { 567 unsigned char *ptail = ZIPLIST_ENTRY_TAIL(zl); 568 if (ptail[0] != ZIP_END) { 569 prevlen = zipRawEntryLength(ptail); 570 } 571 } 572 //看这个要存如的数据是否可以存储成int类型,如果可以,返回value及encoding,如果不可以,则存入原字符串 573 /* See if the entry can be encoded */ 574 if (zipTryEncoding(s,slen,&value,&encoding)) { 575 /* 'encoding' is set to the appropriate integer encoding */ 576 reqlen = zipIntSize(encoding); 577 } else { 578 /* 'encoding' is untouched, however zipEncodeLength will use the 579 * string length to figure out how to encode it. */ 580 reqlen = slen; 581 } 582 //得到当前entry头部两个字段需要的字节数,加上存入数据的长度,即得到整个entry所需要的字节数 583 /* We need space for both the length of the previous entry and 584 * the length of the payload. */ 585 reqlen += zipPrevEncodeLength(NULL,prevlen); 586 reqlen += zipEncodeLength(NULL,encoding,slen); 587 588 /* When the insert position is not equal to the tail, we need to 589 * make sure that the next entry can hold this entry's length in 590 * its prevlen field. */ 591 //得到下个entry的prevlen字段需要的自己数和当前字节数的差 592 nextdiff = (p[0] != ZIP_END) ? zipPrevLenByteDiff(p,reqlen) : 0; 593 594 //所以共需要增加的数据长度为reqlen+nextdiff 595 /* Store offset because a realloc may change the address of zl. */ 596 offset = p-zl; 597 zl = ziplistResize(zl,curlen+reqlen+nextdiff); 598 p = zl+offset; 599 600 /* Apply memory move when necessary and update tail offset. */ 601 if (p[0] != ZIP_END) { 602 /* Subtract one because of the ZIP_END bytes */ 603 //为新entry流出reqlen长度的空隙 604 memmove(p+reqlen,p-nextdiff,curlen-offset-1+nextdiff); 605 606 /* Encode this entry's raw length in the next entry. */ 607 //修改下个entry的prevlen字段 608 zipPrevEncodeLength(p+reqlen,reqlen); 609 //更新zl的offset字段 610 /* Update offset for tail */ 611 ZIPLIST_TAIL_OFFSET(zl) += reqlen; 612 613 /* When the tail contains more than one entry, we need to take 614 * "nextdiff" in account as well. Otherwise, a change in the 615 * size of prevlen doesn't have an effect on the *tail* offset. */ 616 //得到下一个entry的结构体 617 tail = zipEntry(p+reqlen); 618 if (p[reqlen+tail.headersize+tail.len] != ZIP_END) 619 ZIPLIST_TAIL_OFFSET(zl) += nextdiff; 620 //插入的entry在zl尾部,更新offerset 621 } else { 622 /* This element will be the new tail. */ 623 ZIPLIST_TAIL_OFFSET(zl) = p-zl; 624 } 625 626 /* When nextdiff != 0, the raw length of the next entry has changed, so 627 * we need to cascade the update throughout the ziplist */ 628 if (nextdiff != 0) { 629 offset = p-zl; 630 zl = __ziplistCascadeUpdate(zl,p+reqlen); 631 p = zl+offset; 632 } 633 634 //把entry写入空隙中 635 /* Write the entry */ 636 p += zipPrevEncodeLength(p,prevlen); 637 p += zipEncodeLength(p,encoding,slen); 638 if (ZIP_IS_STR(encoding)) { 639 memcpy(p,s,slen); 640 } else { 641 zipSaveInteger(p,value,encoding); 642 } 643 //更新zl的zllen字段 644 ZIPLIST_INCR_LENGTH(zl,1); 645 return zl; 646 } 647 648 //向zl中push一个s开头长度slen的字符串 649 unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where) { 650 unsigned char *p; 651 p = (where == ZIPLIST_HEAD) ? ZIPLIST_ENTRY_HEAD(zl) : ZIPLIST_ENTRY_END(zl); 652 return __ziplistInsert(zl,p,s,slen); 653 } 654 655 656 657 //遍历zl,找到index所在位置的头指针,如果超出范围则返回NULL指针 658 /* Returns an offset to use for iterating with ziplistNext. When the given 659 * index is negative, the list is traversed back to front. When the list 660 * doesn't contain an element at the provided index, NULL is returned. */ 661 unsigned char *ziplistIndex(unsigned char *zl, int index) { 662 unsigned char *p; 663 zlentry entry; 664 if (index < 0) { 665 index = (-index)-1; 666 p = ZIPLIST_ENTRY_TAIL(zl); 667 if (p[0] != ZIP_END) { 668 entry = zipEntry(p); 669 while (entry.prevrawlen > 0 && index--) { 670 p -= entry.prevrawlen; 671 entry = zipEntry(p); 672 } 673 } 674 } else { 675 p = ZIPLIST_ENTRY_HEAD(zl); 676 while (p[0] != ZIP_END && index--) { 677 p += zipRawEntryLength(p); 678 } 679 } 680 return (p[0] == ZIP_END || index > 0) ? NULL : p; 681 } 682 683 //输入当前entry的头指针p,得到下一个entry的头指针,如果不存在,返回NULL 684 /* Return pointer to next entry in ziplist. 685 * 686 * zl is the pointer to the ziplist 687 * p is the pointer to the current element 688 * 689 * The element after 'p' is returned, otherwise NULL if we are at the end. */ 690 unsigned char *ziplistNext(unsigned char *zl, unsigned char *p) { 691 ((void) zl); 692 693 /* "p" could be equal to ZIP_END, caused by ziplistDelete, 694 * and we should return NULL. Otherwise, we should return NULL 695 * when the *next* element is ZIP_END (there is no next entry). */ 696 if (p[0] == ZIP_END) { 697 return NULL; 698 } else { 699 p = p+zipRawEntryLength(p); 700 return (p[0] == ZIP_END) ? NULL : p; 701 } 702 } 703 704 705 //输入zl中的一个entry的头指针p,返回其前一个元素的头指针,如果不存在,返回NULL 706 /* Return pointer to previous entry in ziplist. */ 707 unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p) { 708 zlentry entry; 709 710 /* Iterating backwards from ZIP_END should return the tail. When "p" is 711 * equal to the first element of the list, we're already at the head, 712 * and should return NULL. */ 713 if (p[0] == ZIP_END) { 714 p = ZIPLIST_ENTRY_TAIL(zl); 715 return (p[0] == ZIP_END) ? NULL : p; 716 } else if (p == ZIPLIST_ENTRY_HEAD(zl)) { 717 return NULL; 718 } else { 719 entry = zipEntry(p); 720 assert(entry.prevrawlen > 0); 721 return p-entry.prevrawlen; 722 } 723 } 724 725 //根据entry的头指针,提取其中的内容,如果p为NULL或者是尾节点,则返回0,否则返回1. 如果p entry中的数据为字符串,则存入*sstr,如果是数字,存入*sval 726 /* Get entry pointer to by 'p' and store in either 'e' or 'v' depending 727 * on the encoding of the entry. 'e' is always set to NULL to be able 728 * to find out whether the string pointer or the integer value was set. 729 * Return 0 if 'p' points to the end of the zipmap, 1 otherwise. */ 730 unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *slen, long long *sval) { 731 zlentry entry; 732 if (p == NULL || p[0] == ZIP_END) return 0; 733 if (sstr) *sstr = NULL; 734 735 entry = zipEntry(p); 736 if (ZIP_IS_STR(entry.encoding)) { 737 if (sstr) { 738 *slen = entry.len; 739 *sstr = p+entry.headersize; 740 } 741 } else { 742 if (sval) { 743 *sval = zipLoadInteger(p+entry.headersize,entry.encoding); 744 } 745 } 746 return 1; 747 } 748 749 //在zl的p位置存入元素s,s的长度为slen 750 /* Insert an entry at "p". */ 751 unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) { 752 return __ziplistInsert(zl,p,s,slen); 753 } 754 755 //删除*p指向entry,并返回下一个entry的头指针*p 756 /* Delete a single entry from the ziplist, pointed to by *p. 757 * Also update *p in place, to be able to iterate over the 758 * ziplist, while deleting entries. */ 759 unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) { 760 unsigned int offset = *p-zl; 761 zl = __ziplistDelete(zl,*p,1); 762 763 /* Store pointer to current element in p, because ziplistDelete will 764 * do a realloc which might result in a different "zl"-pointer. 765 * When the delete direction is back to front, we might delete the last 766 * entry and end up with "p" pointing to ZIP_END, so check this. */ 767 *p = zl+offset; 768 return zl; 769 } 770 771 //从index,开始,删除num个entry 772 /* Delete a range of entries from the ziplist. */ 773 unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num) { 774 unsigned char *p = ziplistIndex(zl,index); 775 return (p == NULL) ? zl : __ziplistDelete(zl,p,num); 776 } 777 778 //把p开始的entry所包含的value和长度为slen的sstr串作比较 779 /* Compare entry pointer to by 'p' with 'entry'. Return 1 if equal. */ 780 unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int slen) { 781 zlentry entry; 782 unsigned char sencoding; 783 long long zval, sval; 784 if (p[0] == ZIP_END) return 0; 785 786 entry = zipEntry(p); 787 if (ZIP_IS_STR(entry.encoding)) { 788 /* Raw compare */ 789 if (entry.len == slen) { 790 return memcmp(p+entry.headersize,sstr,slen) == 0; 791 } else { 792 return 0; 793 } 794 } else { 795 /* Try to compare encoded values */ 796 if (zipTryEncoding(sstr,slen,&sval,&sencoding)) { 797 if (entry.encoding == sencoding) { 798 zval = zipLoadInteger(p+entry.headersize,entry.encoding); 799 return zval == sval; 800 } 801 } 802 } 803 return 0; 804 } 805 806 //返回zl所包含的entry数量,如果数量小于2字节能表示的数量【UINT16_MAX】直接返回zl第三段头信息,否则做遍历count数量 807 /* Return length of ziplist. */ 808 unsigned int ziplistLen(unsigned char *zl) { 809 unsigned int len = 0; 810 if (ZIPLIST_LENGTH(zl) < UINT16_MAX) { 811 len = ZIPLIST_LENGTH(zl); 812 } else { 813 unsigned char *p = zl+ZIPLIST_HEADER_SIZE; 814 while (*p != ZIP_END) { 815 p += zipRawEntryLength(p); 816 len++; 817 } 818 819 /* Re-store length if small enough */ 820 if (len < UINT16_MAX) ZIPLIST_LENGTH(zl) = len; 821 } 822 return len; 823 } 824 825 //得到zl整个占用的内存大小,单位bytes 826 /* Return size in bytes of ziplist. */ 827 unsigned int ziplistSize(unsigned char *zl) { 828 return ZIPLIST_BYTES(zl); 829 } 830 831 //把zl tostring了~ 832 void ziplistRepr(unsigned char *zl) { 833 unsigned char *p; 834 int index = 0; 835 zlentry entry; 836 837 printf( 838 "{total bytes %d} " 839 "{length %u}\n" 840 "{tail offset %u}\n", 841 ZIPLIST_BYTES(zl), 842 ZIPLIST_LENGTH(zl), 843 ZIPLIST_TAIL_OFFSET(zl)); 844 p = ZIPLIST_ENTRY_HEAD(zl); 845 while(*p != ZIP_END) { 846 entry = zipEntry(p); 847 printf( 848 "{" 849 "addr 0x%08lx, " 850 "index %2d, " 851 "offset %5ld, " 852 "rl: %5u, " 853 "hs %2u, " 854 "pl: %5u, " 855 "pls: %2u, " 856 "payload %5u" 857 "} ", 858 (long unsigned)p, 859 index, 860 (unsigned long) (p-zl), 861 entry.headersize+entry.len, 862 entry.headersize, 863 entry.prevrawlen, 864 entry.prevrawlensize, 865 entry.len); 866 p += entry.headersize; 867 if (ZIP_IS_STR(entry.encoding)) { 868 if (entry.len > 40) { 869 if (fwrite(p,40,1,stdout) == 0) perror("fwrite"); 870 printf("..."); 871 } else { 872 if (entry.len && 873 fwrite(p,entry.len,1,stdout) == 0) perror("fwrite"); 874 } 875 } else { 876 printf("%lld", (long long) zipLoadInteger(p,entry.encoding)); 877 } 878 printf("\n"); 879 p += entry.len; 880 index++; 881 } 882 printf("{end}\n\n"); 883 } 884 885 #ifdef ZIPLIST_TEST_MAIN 886 #include <sys/time.h> 887 #include "adlist.h" 888 #include "sds.h" 889 890 #define debug(f, ...) { if (DEBUG) printf(f, __VA_ARGS__); } 891 892 unsigned char *createList() { 893 unsigned char *zl = ziplistNew(); 894 zl = ziplistPush(zl, (unsigned char*)"foo", 3, ZIPLIST_TAIL); 895 zl = ziplistPush(zl, (unsigned char*)"quux", 4, ZIPLIST_TAIL); 896 zl = ziplistPush(zl, (unsigned char*)"hello", 5, ZIPLIST_HEAD); 897 zl = ziplistPush(zl, (unsigned char*)"1024", 4, ZIPLIST_TAIL); 898 return zl; 899 } 900 901 unsigned char *createIntList() { 902 unsigned char *zl = ziplistNew(); 903 char buf[32]; 904 905 sprintf(buf, "100"); 906 zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL); 907 sprintf(buf, "128000"); 908 zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL); 909 sprintf(buf, "-100"); 910 zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_HEAD); 911 sprintf(buf, "4294967296"); 912 zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_HEAD); 913 sprintf(buf, "non integer"); 914 zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL); 915 sprintf(buf, "much much longer non integer"); 916 zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL); 917 return zl; 918 } 919 920 long long usec(void) { 921 struct timeval tv; 922 gettimeofday(&tv,NULL); 923 return (((long long)tv.tv_sec)*1000000)+tv.tv_usec; 924 } 925 926 void stress(int pos, int num, int maxsize, int dnum) { 927 int i,j,k; 928 unsigned char *zl; 929 char posstr[2][5] = { "HEAD", "TAIL" }; 930 long long start; 931 for (i = 0; i < maxsize; i+=dnum) { 932 zl = ziplistNew(); 933 for (j = 0; j < i; j++) { 934 zl = ziplistPush(zl,(unsigned char*)"quux",4,ZIPLIST_TAIL); 935 } 936 937 /* Do num times a push+pop from pos */ 938 start = usec(); 939 for (k = 0; k < num; k++) { 940 zl = ziplistPush(zl,(unsigned char*)"quux",4,pos); 941 zl = ziplistDeleteRange(zl,0,1); 942 } 943 printf("List size: %8d, bytes: %8d, %dx push+pop (%s): %6lld usec\n", 944 i,ZIPLIST_BYTES(zl),num,posstr[pos],usec()-start); 945 zfree(zl); 946 } 947 } 948 949 void pop(unsigned char *zl, int where) { 950 unsigned char *p, *vstr; 951 unsigned int vlen; 952 long long vlong; 953 954 p = ziplistIndex(zl,where == ZIPLIST_HEAD ? 0 : -1); 955 if (ziplistGet(p,&vstr,&vlen,&vlong)) { 956 if (where == ZIPLIST_HEAD) 957 printf("Pop head: "); 958 else 959 printf("Pop tail: "); 960 961 if (vstr) 962 if (vlen && fwrite(vstr,vlen,1,stdout) == 0) perror("fwrite"); 963 else 964 printf("%lld", vlong); 965 966 printf("\n"); 967 ziplistDeleteRange(zl,-1,1); 968 } else { 969 printf("ERROR: Could not pop\n"); 970 exit(1); 971 } 972 } 973 974 int randstring(char *target, unsigned int min, unsigned int max) { 975 int p, len = min+rand()%(max-min+1); 976 int minval, maxval; 977 switch(rand() % 3) { 978 case 0: 979 minval = 0; 980 maxval = 255; 981 break; 982 case 1: 983 minval = 48; 984 maxval = 122; 985 break; 986 case 2: 987 minval = 48; 988 maxval = 52; 989 break; 990 default: 991 assert(NULL); 992 } 993 994 while(p < len) 995 target[p++] = minval+rand()%(maxval-minval+1); 996 return len; 997 } 998 999 int main(int argc, char **argv) { 1000 unsigned char *zl, *p; 1001 unsigned char *entry; 1002 unsigned int elen; 1003 long long value; 1004 1005 /* If an argument is given, use it as the random seed. */ 1006 if (argc == 2) 1007 srand(atoi(argv[1])); 1008 1009 zl = createIntList(); 1010 ziplistRepr(zl); 1011 1012 zl = createList(); 1013 ziplistRepr(zl); 1014 1015 pop(zl,ZIPLIST_TAIL); 1016 ziplistRepr(zl); 1017 1018 pop(zl,ZIPLIST_HEAD); 1019 ziplistRepr(zl); 1020 1021 pop(zl,ZIPLIST_TAIL); 1022 ziplistRepr(zl); 1023 1024 pop(zl,ZIPLIST_TAIL); 1025 ziplistRepr(zl); 1026 1027 printf("Get element at index 3:\n"); 1028 { 1029 zl = createList(); 1030 p = ziplistIndex(zl, 3); 1031 if (!ziplistGet(p, &entry, &elen, &value)) { 1032 printf("ERROR: Could not access index 3\n"); 1033 return 1; 1034 } 1035 if (entry) { 1036 if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); 1037 printf("\n"); 1038 } else { 1039 printf("%lld\n", value); 1040 } 1041 printf("\n"); 1042 } 1043 1044 printf("Get element at index 4 (out of range):\n"); 1045 { 1046 zl = createList(); 1047 p = ziplistIndex(zl, 4); 1048 if (p == NULL) { 1049 printf("No entry\n"); 1050 } else { 1051 printf("ERROR: Out of range index should return NULL, returned offset: %ld\n", p-zl); 1052 return 1; 1053 } 1054 printf("\n"); 1055 } 1056 1057 printf("Get element at index -1 (last element):\n"); 1058 { 1059 zl = createList(); 1060 p = ziplistIndex(zl, -1); 1061 if (!ziplistGet(p, &entry, &elen, &value)) { 1062 printf("ERROR: Could not access index -1\n"); 1063 return 1; 1064 } 1065 if (entry) { 1066 if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); 1067 printf("\n"); 1068 } else { 1069 printf("%lld\n", value); 1070 } 1071 printf("\n"); 1072 } 1073 1074 printf("Get element at index -4 (first element):\n"); 1075 { 1076 zl = createList(); 1077 p = ziplistIndex(zl, -4); 1078 if (!ziplistGet(p, &entry, &elen, &value)) { 1079 printf("ERROR: Could not access index -4\n"); 1080 return 1; 1081 } 1082 if (entry) { 1083 if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); 1084 printf("\n"); 1085 } else { 1086 printf("%lld\n", value); 1087 } 1088 printf("\n"); 1089 } 1090 1091 printf("Get element at index -5 (reverse out of range):\n"); 1092 { 1093 zl = createList(); 1094 p = ziplistIndex(zl, -5); 1095 if (p == NULL) { 1096 printf("No entry\n"); 1097 } else { 1098 printf("ERROR: Out of range index should return NULL, returned offset: %ld\n", p-zl); 1099 return 1; 1100 } 1101 printf("\n"); 1102 } 1103 1104 printf("Iterate list from 0 to end:\n"); 1105 { 1106 zl = createList(); 1107 p = ziplistIndex(zl, 0); 1108 while (ziplistGet(p, &entry, &elen, &value)) { 1109 printf("Entry: "); 1110 if (entry) { 1111 if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); 1112 } else { 1113 printf("%lld", value); 1114 } 1115 p = ziplistNext(zl,p); 1116 printf("\n"); 1117 } 1118 printf("\n"); 1119 } 1120 1121 printf("Iterate list from 1 to end:\n"); 1122 { 1123 zl = createList(); 1124 p = ziplistIndex(zl, 1); 1125 while (ziplistGet(p, &entry, &elen, &value)) { 1126 printf("Entry: "); 1127 if (entry) { 1128 if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); 1129 } else { 1130 printf("%lld", value); 1131 } 1132 p = ziplistNext(zl,p); 1133 printf("\n"); 1134 } 1135 printf("\n"); 1136 } 1137 1138 printf("Iterate list from 2 to end:\n"); 1139 { 1140 zl = createList(); 1141 p = ziplistIndex(zl, 2); 1142 while (ziplistGet(p, &entry, &elen, &value)) { 1143 printf("Entry: "); 1144 if (entry) { 1145 if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); 1146 } else { 1147 printf("%lld", value); 1148 } 1149 p = ziplistNext(zl,p); 1150 printf("\n"); 1151 } 1152 printf("\n"); 1153 } 1154 1155 printf("Iterate starting out of range:\n"); 1156 { 1157 zl = createList(); 1158 p = ziplistIndex(zl, 4); 1159 if (!ziplistGet(p, &entry, &elen, &value)) { 1160 printf("No entry\n"); 1161 } else { 1162 printf("ERROR\n"); 1163 } 1164 printf("\n"); 1165 } 1166 1167 printf("Iterate from back to front:\n"); 1168 { 1169 zl = createList(); 1170 p = ziplistIndex(zl, -1); 1171 while (ziplistGet(p, &entry, &elen, &value)) { 1172 printf("Entry: "); 1173 if (entry) { 1174 if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); 1175 } else { 1176 printf("%lld", value); 1177 } 1178 p = ziplistPrev(zl,p); 1179 printf("\n"); 1180 } 1181 printf("\n"); 1182 } 1183 1184 printf("Iterate from back to front, deleting all items:\n"); 1185 { 1186 zl = createList(); 1187 p = ziplistIndex(zl, -1); 1188 while (ziplistGet(p, &entry, &elen, &value)) { 1189 printf("Entry: "); 1190 if (entry) { 1191 if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); 1192 } else { 1193 printf("%lld", value); 1194 } 1195 zl = ziplistDelete(zl,&p); 1196 p = ziplistPrev(zl,p); 1197 printf("\n"); 1198 } 1199 printf("\n"); 1200 } 1201 1202 printf("Delete inclusive range 0,0:\n"); 1203 { 1204 zl = createList(); 1205 zl = ziplistDeleteRange(zl, 0, 1); 1206 ziplistRepr(zl); 1207 } 1208 1209 printf("Delete inclusive range 0,1:\n"); 1210 { 1211 zl = createList(); 1212 zl = ziplistDeleteRange(zl, 0, 2); 1213 ziplistRepr(zl); 1214 } 1215 1216 printf("Delete inclusive range 1,2:\n"); 1217 { 1218 zl = createList(); 1219 zl = ziplistDeleteRange(zl, 1, 2); 1220 ziplistRepr(zl); 1221 } 1222 1223 printf("Delete with start index out of range:\n"); 1224 { 1225 zl = createList(); 1226 zl = ziplistDeleteRange(zl, 5, 1); 1227 ziplistRepr(zl); 1228 } 1229 1230 printf("Delete with num overflow:\n"); 1231 { 1232 zl = createList(); 1233 zl = ziplistDeleteRange(zl, 1, 5); 1234 ziplistRepr(zl); 1235 } 1236 1237 printf("Delete foo while iterating:\n"); 1238 { 1239 zl = createList(); 1240 p = ziplistIndex(zl,0); 1241 while (ziplistGet(p,&entry,&elen,&value)) { 1242 if (entry && strncmp("foo",(char*)entry,elen) == 0) { 1243 printf("Delete foo\n"); 1244 zl = ziplistDelete(zl,&p); 1245 } else { 1246 printf("Entry: "); 1247 if (entry) { 1248 if (elen && fwrite(entry,elen,1,stdout) == 0) 1249 perror("fwrite"); 1250 } else { 1251 printf("%lld",value); 1252 } 1253 p = ziplistNext(zl,p); 1254 printf("\n"); 1255 } 1256 } 1257 printf("\n"); 1258 ziplistRepr(zl); 1259 } 1260 1261 printf("Regression test for >255 byte strings:\n"); 1262 { 1263 char v1[257],v2[257]; 1264 memset(v1,'x',256); 1265 memset(v2,'y',256); 1266 zl = ziplistNew(); 1267 zl = ziplistPush(zl,(unsigned char*)v1,strlen(v1),ZIPLIST_TAIL); 1268 zl = ziplistPush(zl,(unsigned char*)v2,strlen(v2),ZIPLIST_TAIL); 1269 1270 /* Pop values again and compare their value. */ 1271 p = ziplistIndex(zl,0); 1272 assert(ziplistGet(p,&entry,&elen,&value)); 1273 assert(strncmp(v1,(char*)entry,elen) == 0); 1274 p = ziplistIndex(zl,1); 1275 assert(ziplistGet(p,&entry,&elen,&value)); 1276 assert(strncmp(v2,(char*)entry,elen) == 0); 1277 printf("SUCCESS\n\n"); 1278 } 1279 1280 printf("Create long list and check indices:\n"); 1281 { 1282 zl = ziplistNew(); 1283 char buf[32]; 1284 int i,len; 1285 for (i = 0; i < 1000; i++) { 1286 len = sprintf(buf,"%d",i); 1287 zl = ziplistPush(zl,(unsigned char*)buf,len,ZIPLIST_TAIL); 1288 } 1289 for (i = 0; i < 1000; i++) { 1290 p = ziplistIndex(zl,i); 1291 assert(ziplistGet(p,NULL,NULL,&value)); 1292 assert(i == value); 1293 1294 p = ziplistIndex(zl,-i-1); 1295 assert(ziplistGet(p,NULL,NULL,&value)); 1296 assert(999-i == value); 1297 } 1298 printf("SUCCESS\n\n"); 1299 } 1300 1301 printf("Compare strings with ziplist entries:\n"); 1302 { 1303 zl = createList(); 1304 p = ziplistIndex(zl,0); 1305 if (!ziplistCompare(p,(unsigned char*)"hello",5)) { 1306 printf("ERROR: not \"hello\"\n"); 1307 return 1; 1308 } 1309 if (ziplistCompare(p,(unsigned char*)"hella",5)) { 1310 printf("ERROR: \"hella\"\n"); 1311 return 1; 1312 } 1313 1314 p = ziplistIndex(zl,3); 1315 if (!ziplistCompare(p,(unsigned char*)"1024",4)) { 1316 printf("ERROR: not \"1024\"\n"); 1317 return 1; 1318 } 1319 if (ziplistCompare(p,(unsigned char*)"1025",4)) { 1320 printf("ERROR: \"1025\"\n"); 1321 return 1; 1322 } 1323 printf("SUCCESS\n\n"); 1324 } 1325 1326 printf("Stress with random payloads of different encoding:\n"); 1327 { 1328 int i,j,len,where; 1329 unsigned char *p; 1330 char buf[1024]; 1331 int buflen; 1332 list *ref; 1333 listNode *refnode; 1334 1335 /* Hold temp vars from ziplist */ 1336 unsigned char *sstr; 1337 unsigned int slen; 1338 long long sval; 1339 1340 for (i = 0; i < 20000; i++) { 1341 zl = ziplistNew(); 1342 ref = listCreate(); 1343 listSetFreeMethod(ref,sdsfree); 1344 len = rand() % 256; 1345 1346 /* Create lists */ 1347 for (j = 0; j < len; j++) { 1348 where = (rand() & 1) ? ZIPLIST_HEAD : ZIPLIST_TAIL; 1349 if (rand() % 2) { 1350 buflen = randstring(buf,1,sizeof(buf)-1); 1351 } else { 1352 switch(rand() % 3) { 1353 case 0: 1354 buflen = sprintf(buf,"%lld",(0LL + rand()) >> 20); 1355 break; 1356 case 1: 1357 buflen = sprintf(buf,"%lld",(0LL + rand())); 1358 break; 1359 case 2: 1360 buflen = sprintf(buf,"%lld",(0LL + rand()) << 20); 1361 break; 1362 default: 1363 assert(NULL); 1364 } 1365 } 1366 1367 /* Add to ziplist */ 1368 zl = ziplistPush(zl, (unsigned char*)buf, buflen, where); 1369 1370 /* Add to reference list */ 1371 if (where == ZIPLIST_HEAD) { 1372 listAddNodeHead(ref,sdsnewlen(buf, buflen)); 1373 } else if (where == ZIPLIST_TAIL) { 1374 listAddNodeTail(ref,sdsnewlen(buf, buflen)); 1375 } else { 1376 assert(NULL); 1377 } 1378 } 1379 1380 assert(listLength(ref) == ziplistLen(zl)); 1381 for (j = 0; j < len; j++) { 1382 /* Naive way to get elements, but similar to the stresser 1383 * executed from the Tcl test suite. */ 1384 p = ziplistIndex(zl,j); 1385 refnode = listIndex(ref,j); 1386 1387 assert(ziplistGet(p,&sstr,&slen,&sval)); 1388 if (sstr == NULL) { 1389 buflen = sprintf(buf,"%lld",sval); 1390 } else { 1391 buflen = slen; 1392 memcpy(buf,sstr,buflen); 1393 buf[buflen] = '\0'; 1394 } 1395 assert(memcmp(buf,listNodeValue(refnode),buflen) == 0); 1396 } 1397 zfree(zl); 1398 listRelease(ref); 1399 } 1400 printf("SUCCESS\n\n"); 1401 } 1402 1403 printf("Stress with variable ziplist size:\n"); 1404 { 1405 stress(ZIPLIST_HEAD,100000,16384,256); 1406 stress(ZIPLIST_TAIL,100000,16384,256); 1407 } 1408 1409 return 0; 1410 } 1411 1412 #endif