LevelDB的源码阅读（三） Get操作

在Linux上leveldb的安装和使用中我们写了这么一段测试代码，内容以及输出结果如下：

#include <iostream>
#include <string>
#include <assert.h>    
#include "leveldb/db.h"    

using namespace std;

int main(void) 
{       

    leveldb::DB      *db;    
    leveldb::Options  options;    
    options.create_if_missing = true;    

    // open
    leveldb::Status status = leveldb::DB::Open(options,"/tmp/testdb", &db);    
    assert(status.ok());    

    string key = "name";    
    string value = "chenqi";    

    // write
    status = db->Put(leveldb::WriteOptions(), key, value);    
    assert(status.ok());    

    // read
    status = db->Get(leveldb::ReadOptions(), key, &value);    
    assert(status.ok());    

    cout<<value<<endl;    

    // delete
    status = db->Delete(leveldb::WriteOptions(), key);    
    assert(status.ok());        

    status = db->Get(leveldb::ReadOptions(),key, &value);    
    if(!status.ok()) {
        cerr<<key<<"    "<<status.ToString()<<endl;
    } else {
        cout<<key<<"==="<<value<<endl;    
    }   

    // close 
    delete db;    

    return 0;    
}

chenqi
name    NotFound:

Leveldb的读数据入口为db文件夹下db_impl.cc文件中的DBImpl::Get函数，首先获取当前的版本号，然后依次在三个数据源memtable，immutable table,和sst表中进行查找,返回之前再判断一下是否需要启动Compact任务.

Status DBImpl::Get(const ReadOptions& options,
                   const Slice& key,
                   std::string* value) {
  Status s;
  MutexLock l(&mutex_);
  SequenceNumber snapshot;
  if (options.snapshot != NULL) {
    snapshot = reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_;
  } else {
    //获取版本号 
    snapshot = versions_->LastSequence();
  }
  //三个查找源，memtable,immutable,sstable
  MemTable* mem = mem_;
  MemTable* imm = imm_;
  Version* current = versions_->current();
  mem->Ref();
  if (imm != NULL) imm->Ref();
  current->Ref();

  bool have_stat_update = false;
  Version::GetStats stats;

  // Unlock while reading from files and memtables
  {
    mutex_.Unlock();
    // First look in the memtable, then in the immutable memtable (if any).
    LookupKey lkey(key, snapshot);
    if (mem->Get(lkey, value, &s)) {  //在memtable中查找
      // Done
    } else if (imm != NULL && imm->Get(lkey, value, &s)) {//在imutable中查找
      // Done
    } else {
      s = current->Get(options, lkey, value, &stats); //在磁盘文件中查找，当前Version
      have_stat_update = true;
    }
    mutex_.Lock();
  }
  //判断是否需要调度Compact
  if (have_stat_update && current->UpdateStats(stats)) {
    MaybeScheduleCompaction();
  }
  mem->Unref();
  if (imm != NULL) imm->Unref();
  current->Unref();
  return s;
}

首先，按照leveldb代码的惯例线上锁，然后生成一个SequenceNumber作为标记, 后续不管线程会不会被切出去, 结果都要相当于在这个时间点瞬间完成，memtable、immemtable以及Version都由于采用了引用计数, 因此要Ref().快照建立完了, 接下来的操作只会有单纯的读, 可以把锁暂时释放.查询的顺序是先找memtable, 再immemtable, 最后是SSTable.这里调用了db文件夹下dbformat.cc中的LookupKey::LookupKey, 源码内容如下：

LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) {
  size_t usize = user_key.size();
  size_t needed = usize + 13;  // A conservative estimate
  char* dst;
  if (needed <= sizeof(space_)) {
    dst = space_;
  } else {
    dst = new char[needed];
  }
  start_ = dst;
  dst = EncodeVarint32(dst, usize + 8);
  kstart_ = dst;
  memcpy(dst, user_key.data(), usize);
  dst += usize;
  EncodeFixed64(dst, PackSequenceAndType(s, kValueTypeForSeek));
  dst += 8;
  end_ = dst;
}

这个类主要的功能是把输入的key转换成用于查询的key. 比如key是"Sherry", 实际在数据库中的表达可能会是"6Sherry", 6是长度. 这样比对key是否相等时速度会更快.LookupKey格式 = 长度 + key + SequenceNumber + type,这里用了两个tricks:

1.在栈上分配一个200长度的数组, 如果运行时发现长度不够用再从堆上new一个, 可以极大避免内存分配

2.黑科技函数"EncodeVarint32", 一般key的长度不可能用满32bit. 大量很短的Key却要用32bit来描述长度无疑是很浪费的. 这个函数让小数值用更少的空间, 代价是最糟要多花一字节(8bit).EncodeVarint32的代码出现在util文件夹下的coding.cc文件里，源码内容如下： 

char* EncodeVarint32(char* dst, uint32_t v) {
  // Operate on characters as unsigneds
  unsigned char* ptr = reinterpret_cast<unsigned char*>(dst);
  static const int B = 128;
  if (v < (1<<7)) {
    *(ptr++) = v;
  } else if (v < (1<<14)) {
    *(ptr++) = v | B;
    *(ptr++) = v>>7;
  } else if (v < (1<<21)) {
    *(ptr++) = v | B;
    *(ptr++) = (v>>7) | B;
    *(ptr++) = v>>14;
  } else if (v < (1<<28)) {
    *(ptr++) = v | B;
    *(ptr++) = (v>>7) | B;
    *(ptr++) = (v>>14) | B;
    *(ptr++) = v>>21;
  } else {
    *(ptr++) = v | B;
    *(ptr++) = (v>>7) | B;
    *(ptr++) = (v>>14) | B;
    *(ptr++) = (v>>21) | B;
    *(ptr++) = v>>28;
  }
  return reinterpret_cast<char*>(ptr);
}

 现在回到DBImpl::Get函数，memtable和immutable table都是通过内存中的skiplist进行的，磁盘文件的查找是通过db文件夹下version_set.cc中Version::Get来进行的.源码内容如下： 

Status Version::Get(const ReadOptions& options,
                    const LookupKey& k,
                    std::string* value,
                    GetStats* stats) {
  Slice ikey = k.internal_key();
  Slice user_key = k.user_key();
  const Comparator* ucmp = vset_->icmp_.user_comparator();
  Status s;

  stats->seek_file = NULL;
  stats->seek_file_level = -1;
  FileMetaData* last_file_read = NULL;
  int last_file_read_level = -1;

  // We can search level-by-level since entries never hop across
  // levels.  Therefore we are guaranteed that if we find data
  // in an smaller level, later levels are irrelevant.
  //查找用户提供的key可能在的文件，通过各个level的文件的最小值，最大值来判断
  //按层查找
  std::vector<FileMetaData*> tmp;
  FileMetaData* tmp2;
  for (int level = 0; level < config::kNumLevels; level++) {
    size_t num_files = files_[level].size();
    if (num_files == 0) continue;

    // Get the list of files to search in this level
    FileMetaData* const* files = &files_[level][0];
    if (level == 0) {
      // Level-0 files may overlap each other.  Find all files that
      // overlap user_key and process them in order from newest to oldest.
      tmp.reserve(num_files);
      for (uint32_t i = 0; i < num_files; i++) {
        FileMetaData* f = files[i];
        if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 &&
            ucmp->Compare(user_key, f->largest.user_key()) <= 0) {
          tmp.push_back(f);
        }
      }
      if (tmp.empty()) continue;

      std::sort(tmp.begin(), tmp.end(), NewestFirst);
      files = &tmp[0];
      num_files = tmp.size();
    } else {
      // Binary search to find earliest index whose largest key >= ikey.
      //查找用户提供的key可能在的文件
      uint32_t index = FindFile(vset_->icmp_, files_[level], ikey);
      if (index >= num_files) {
        files = NULL;
        num_files = 0;
      } else {
        tmp2 = files[index];
        if (ucmp->Compare(user_key, tmp2->smallest.user_key()) < 0) {
          // All of "tmp2" is past any data for user_key
          files = NULL;
          num_files = 0;
        } else {
          files = &tmp2;
          num_files = 1;
        }
      }
    }

    for (uint32_t i = 0; i < num_files; ++i) {
      if (last_file_read != NULL && stats->seek_file == NULL) {
        // We have had more than one seek for this read.  Charge the 1st file.
        stats->seek_file = last_file_read;
        stats->seek_file_level = last_file_read_level;
      }

      FileMetaData* f = files[i];
      last_file_read = f;
      last_file_read_level = level;

      Saver saver;
      saver.state = kNotFound;
      saver.ucmp = ucmp;
      saver.user_key = user_key;
      saver.value = value;
      //在table_cache中查找key对应的value
      s = vset_->table_cache_->Get(options, f->number, f->file_size,
                                   ikey, &saver, SaveValue);
      if (!s.ok()) {
        return s;
      }
      switch (saver.state) {
        case kNotFound:
          break;      // Keep searching in other files
        case kFound:
          return s;
        case kDeleted:
          s = Status::NotFound(Slice());  // Use empty error message for speed
          return s;
        case kCorrupt:
          s = Status::Corruption("corrupted key for ", user_key);
          return s;
      }
    }
  }

  return Status::NotFound(Slice());  // Use an empty error message for speed
}

 FindFile源码内容如下：

int FindFile(const InternalKeyComparator& icmp,
             const std::vector<FileMetaData*>& files,
             const Slice& key) {
  uint32_t left = 0;
  uint32_t right = files.size();
  while (left < right) {
    uint32_t mid = (left + right) / 2;
    const FileMetaData* f = files[mid];
    if (icmp.InternalKeyComparator::Compare(f->largest.Encode(), key) < 0) {
      // Key at "mid.largest" is < "target".  Therefore all
      // files at or before "mid" are uninteresting.
      left = mid + 1;
    } else {
      // Key at "mid.largest" is >= "target".  Therefore all files
      // after "mid" are uninteresting.
      right = mid;
    }
  }
  return right;
}

Version::Get函数首先查找key可能存在的sst表，然后调用table_cache->Get进行查找。即对SSTable的查询就是对table_cache_的查询, 这个cache是不可取消的, 解决了什么问题呢?

LevelDB的数据库"文件"是一个文件夹, 里面包含大量的文件. 这是把复杂度甩锅给操作系统的做法, 但很多系统资源是有限的. 比如, file handle(文件句柄). 一个程序如果开了1W个file handle会浪费大量资源. 这里做个LRU cache, 只有常用的SSTable才会开一个活跃的file handle.

另外就是索引的问题. LSMT是没有主索引的, 只有在各个SSTable内有微缩版索引. 所以, 最最优的情况下也需要2次硬盘读写. 第一张SSTable就存着key, 先读微型索引, 然后二分法找到具体位置, 再读value.

TableCache把热点SSTable的微型索引预先放在内存里, 这样只要1次硬盘读取就能取到key. 这个优化对于LSMT的数据库来说尤为重要, 因为很可能会不止查询一张SSTable. 情况会劣化非常快.

总结, TableCache既承担管理资源(file handle)的作用, 又加速索引的读取.

TableCache的实现在db文件夹下table_cache.cc中，源码内容如下： 

Status TableCache::Get(const ReadOptions& options,
                       uint64_t file_number,
                       uint64_t file_size,
                       const Slice& k,
                       void* arg,
                       void (*saver)(void*, const Slice&, const Slice&)) {
  Cache::Handle* handle = NULL;
  Status s = FindTable(file_number, file_size, &handle);//查找table，没有则新建table结构并插入table_cache
  if (s.ok()) {
    Table* t = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
    s = t->InternalGet(options, k, arg, saver);  //在table中查找
    cache_->Release(handle);
  }
  return s;
}

 该函数流程很简单，先从table_cache中获取Table结构，没有则新建Table结构加入table_cache,然后调用Table::InternalGet在具体的sst表中查找.需要注意的是，在util文件夹下查看cache.cc可以看到 

virtual Handle* Insert(const Slice& key, void* value, size_t charge,
                         void (*deleter)(const Slice& key, void* value)) {
    const uint32_t hash = HashSlice(key);
    return shard_[Shard(hash)].Insert(key, hash, value, charge, deleter);
  }
  virtual Handle* Lookup(const Slice& key) {
    const uint32_t hash = HashSlice(key);
    return shard_[Shard(hash)].Lookup(key, hash);
  }

 这个hash table做了两遍hash, 先把key分片一遍, 然后再扔给真正的hash table cache(有锁)去lookup.这么做的逻辑是可以减少锁的使用率和提升并发.

进一步查看table_cache.cc中的Table::InternalGet函数： 

Status Table::InternalGet(const ReadOptions& options, const Slice& k,
                          void* arg,
                          void (*saver)(void*, const Slice&, const Slice&)) {
  Status s;
  Iterator* iiter = rep_->index_block->NewIterator(rep_->options.comparator);
  iiter->Seek(k);//在索引中找，是否存在某个块可能包含这个key
  if (iiter->Valid()) {
    Slice handle_value = iiter->value();
    FilterBlockReader* filter = rep_->filter;
    BlockHandle handle;
    if (filter != NULL &&
        handle.DecodeFrom(&handle_value).ok() &&
        !filter->KeyMayMatch(handle.offset(), k)) {
      // Not found
    } else {
      //在具体的block中找
      Iterator* block_iter = BlockReader(this, options, iiter->value());
      block_iter->Seek(k);
      if (block_iter->Valid()) {
        (*saver)(arg, block_iter->key(), block_iter->value());
      }
      s = block_iter->status();
      delete block_iter;
    }
  }
  if (s.ok()) {
    s = iiter->status();
  }
  delete iiter;
  return s;
}

Table::Get函数先在table的indexblock中查找该key所处的block,然后利用Bloom Filter来过滤，最后在具体的block中查找。在查找过程中使用了Iterator机制。

 总体来说，leveldb中Get操作的流程可以用下图来说明： 

 参考文献：

1.http://blog.csdn.net/joeyon1985/article/details/47154249

2.http://masutangu.com/2017/06/leveldb_1/

3.https://zhuanlan.zhihu.com/jimderestaurant?topic=LevelDB