path: root/src/ceph/0006-librbd-LRU-policy-based-eviction.patch

From b233d6540160c8bc5cc25b870c2140fa48776fa6 Mon Sep 17 00:00:00 2001
From: Dehao Shang <dehao.shang@intel.com>
Date: Mon, 6 Aug 2018 22:42:38 +0800
Subject: [PATCH 06/10] librbd: LRU policy based eviction

Signed-off-by: Dehao Shang <dehao.shang@intel.com>
Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
---
 src/tools/rbd_cache/ObjectCacheStore.cc |  73 ++++++++-----
 src/tools/rbd_cache/ObjectCacheStore.h  |  14 +--
 src/tools/rbd_cache/Policy.hpp          |  22 ++++
 src/tools/rbd_cache/SimplePolicy.hpp    | 180 ++++++++++++++++++++++++++++++++
 4 files changed, 254 insertions(+), 35 deletions(-)
 create mode 100644 src/tools/rbd_cache/Policy.hpp
 create mode 100644 src/tools/rbd_cache/SimplePolicy.hpp

diff --git a/src/tools/rbd_cache/ObjectCacheStore.cc b/src/tools/rbd_cache/ObjectCacheStore.cc
index 2a87469..b39fe66 100644
--- a/src/tools/rbd_cache/ObjectCacheStore.cc
+++ b/src/tools/rbd_cache/ObjectCacheStore.cc
@@ -14,12 +14,12 @@ namespace cache {
 
 ObjectCacheStore::ObjectCacheStore(CephContext *cct, ContextWQ* work_queue)
       : m_cct(cct), m_work_queue(work_queue),
-        m_cache_table_lock("rbd::cache::ObjectCacheStore"),
         m_rados(new librados::Rados()) {
+  m_policy = new SimplePolicy(4096, 0.9); // TODO
 }
 
 ObjectCacheStore::~ObjectCacheStore() {
-
+  delete m_policy;
 }
 
 int ObjectCacheStore::init(bool reset) {
@@ -43,6 +43,7 @@ int ObjectCacheStore::do_promote(std::string pool_name, std::string object_name)
   int ret = 0;
   std::string cache_file_name =  pool_name + object_name;
 
+  //TODO(): lock on ioctx map
   if (m_ioctxs.find(pool_name) == m_ioctxs.end()) {
     librados::IoCtx* io_ctx = new librados::IoCtx();
     ret = m_rados->ioctx_create(pool_name.c_str(), *io_ctx);
@@ -58,10 +59,8 @@ int ObjectCacheStore::do_promote(std::string pool_name, std::string object_name)
   librados::IoCtx* ioctx = m_ioctxs[pool_name]; 
 
   //promoting: update metadata 
-  {
-    Mutex::Locker locker(m_cache_table_lock);
-    m_cache_table.emplace(cache_file_name, PROMOTING);
-  }
+  m_policy->update_status(cache_file_name, PROMOTING);
+  assert(PROMOTING == m_policy->get_status(cache_file_name));
 
   librados::bufferlist* read_buf = new librados::bufferlist();
   int object_size = 4096*1024; //TODO(): read config from image metadata
@@ -83,42 +82,60 @@ int ObjectCacheStore::do_promote(std::string pool_name, std::string object_name)
   cache_file.open();
   ret = cache_file.write_object_to_file(*read_buf, object_size);
   
-  assert(m_cache_table.find(cache_file_name) != m_cache_table.end()); 
-
   // update metadata
-  {
-    Mutex::Locker locker(m_cache_table_lock);
-    m_cache_table.emplace(cache_file_name, PROMOTED);
-  }
+  assert(PROMOTING == m_policy->get_status(cache_file_name));
+  m_policy->update_status(cache_file_name, PROMOTED);
+  assert(PROMOTED == m_policy->get_status(cache_file_name));
 
   return ret;
 
 }
  
+// return -1, client need to read data from cluster.
+// return 0,  client directly read data from cache.
 int ObjectCacheStore::lookup_object(std::string pool_name, std::string object_name) {
 
   std::string cache_file_name =  pool_name + object_name;
-  {
-    Mutex::Locker locker(m_cache_table_lock);
-
-    auto it = m_cache_table.find(cache_file_name);
-    if (it != m_cache_table.end()) {
-
-      if (it->second == PROMOTING) {
-        return -1;
-      } else if (it->second == PROMOTED) {
-        return 0;
-      } else {
-        assert(0);
-      }
-    }
+
+  // TODO lookup and return status;
+
+  CACHESTATUS ret;
+  ret = m_policy->lookup_object(cache_file_name);
+
+  switch(ret) {
+    case NONE:
+      return do_promote(pool_name, object_name);
+    case PROMOTING:
+      return -1;
+    case PROMOTED:
+      return 0;
+    default:
+      return -1;
   }
+}
 
-  int ret = do_promote(pool_name, object_name);
+void ObjectCacheStore::evict_thread_body() {
+  int ret;
+  while(m_evict_go) {
+    std::string temp_cache_file;
 
-  return ret;
+    ret = m_policy->evict_object(temp_cache_file);
+    if(ret == 0) {
+      continue;
+    }
+
+    // TODO
+    // delete temp_cache_file file.
+
+    assert(EVICTING == m_policy->get_status(temp_cache_file));
+
+    m_policy->update_status(temp_cache_file, EVICTED);
+
+    assert(NONE == m_policy->get_status(temp_cache_file));
+  }
 }
 
+
 int ObjectCacheStore::shutdown() {
   m_rados->shutdown();
   return 0;
diff --git a/src/tools/rbd_cache/ObjectCacheStore.h b/src/tools/rbd_cache/ObjectCacheStore.h
index db09efa..5118a73 100644
--- a/src/tools/rbd_cache/ObjectCacheStore.h
+++ b/src/tools/rbd_cache/ObjectCacheStore.h
@@ -13,6 +13,7 @@
 #include "librbd/ImageCtx.h"
 #include "librbd/ImageState.h"
 #include "librbd/cache/SharedPersistentObjectCacherFile.h"
+#include "SimplePolicy.hpp"
 
 using librados::Rados;
 using librados::IoCtx;
@@ -39,6 +40,8 @@ class ObjectCacheStore
 
     int lock_cache(std::string vol_name);
 
+    void evict_thread_body();
+
   private:
     int _evict_object();
 
@@ -48,21 +51,18 @@ class ObjectCacheStore
                        librados::bufferlist* read_buf,
                        uint64_t length);
 
-    enum {
-      PROMOTING = 0, 
-      PROMOTED, 
-    };
-
     CephContext *m_cct;
     ContextWQ* m_work_queue;
-    Mutex m_cache_table_lock;
     RadosRef m_rados;
 
-    std::map<std::string, uint8_t> m_cache_table;
 
     std::map<std::string, librados::IoCtx*> m_ioctxs;
 
     librbd::cache::SyncFile *m_cache_file;
+
+    Policy* m_policy;
+
+    bool m_evict_go;
 };
 
 } // namespace rbd
diff --git a/src/tools/rbd_cache/Policy.hpp b/src/tools/rbd_cache/Policy.hpp
new file mode 100644
index 0000000..575c294
--- /dev/null
+++ b/src/tools/rbd_cache/Policy.hpp
@@ -0,0 +1,22 @@
+#ifndef RBD_CACHE_POLICY_HPP
+#define RBD_CACHE_POLICY_HPP
+
+enum CACHESTATUS {
+  NONE = 0,
+  PROMOTING,
+  PROMOTED,
+  EVICTING,
+  EVICTED,
+};
+
+
+class Policy {
+public:
+  Policy(){}
+  virtual ~Policy(){};
+  virtual CACHESTATUS lookup_object(std::string) = 0;
+  virtual int evict_object(std::string&) = 0;
+  virtual void update_status(std::string, CACHESTATUS) = 0;
+  virtual CACHESTATUS get_status(std::string) = 0;
+};
+#endif
diff --git a/src/tools/rbd_cache/SimplePolicy.hpp b/src/tools/rbd_cache/SimplePolicy.hpp
new file mode 100644
index 0000000..a0d8de7
--- /dev/null
+++ b/src/tools/rbd_cache/SimplePolicy.hpp
@@ -0,0 +1,180 @@
+#ifndef RBD_CACHE_SIMPLE_POLICY_HPP
+#define RBD_CACHE_SIMPLE_POLICY_HPP
+
+#include "Policy.hpp"
+#include "include/lru.h"
+#include "common/Mutex.h"
+
+#include <vector>
+#include <unordered_map>
+#include <string>
+
+class SimplePolicy : public Policy {
+public:
+  SimplePolicy(uint64_t block_num, float level)
+    : m_level(level),
+      m_lock("SimplePolicy"),
+      m_entry_count(block_num)
+  {
+
+    Entry m_entries[m_entry_count];
+
+    for(auto &entry : m_entries) {
+      m_free_lru.lru_insert_bot(&entry);
+    }
+  }
+
+  ~SimplePolicy() {}
+
+  CACHESTATUS lookup_object(std::string cache_file_name) {
+    Mutex::Locker locker(m_lock);
+
+    auto entry_it = m_oid_to_entry.find(cache_file_name);
+    if(entry_it == m_oid_to_entry.end()) {
+      return NONE;
+    }
+
+    Entry* entry = entry_it->second;
+
+    LRU* lru;
+    if(entry->status == PROMOTED) {
+      lru = &m_promoted_lru;
+    } else {
+      lru = &m_handing_lru;
+    }
+
+    // touch it
+    lru->lru_remove(entry);
+    lru->lru_insert_top(entry);
+
+    return entry->status;
+  }
+
+  int evict_object(std::string& out_cache_file_name) {
+    Mutex::Locker locker(m_lock);
+
+    // still have enough free space, don't need to evict lru.
+    uint64_t temp_current_size = m_oid_to_entry.size();
+    float temp_current_evict_level = temp_current_size / m_entry_count;
+    if(temp_current_evict_level < m_level) {
+      return 0;
+    }
+
+    // when all entries are USING, PROMOTING or EVICTING, just busy waiting.
+    if(m_promoted_lru.lru_get_size() == 0) {
+      return 0;
+    }
+
+    assert(m_promoted_lru.lru_get_size() != 0);
+
+    // evict one item from promoted lru
+    Entry *entry = reinterpret_cast<Entry*>(m_promoted_lru.lru_get_next_expire());
+    assert(entry != nullptr);
+
+    assert(entry->status == PROMOTED);
+
+    out_cache_file_name = entry->cache_file_name;
+    entry->status = EVICTING;
+
+    m_promoted_lru.lru_remove(entry);
+    m_handing_lru.lru_insert_top(entry);
+
+    return 1;
+  }
+
+  // TODO(): simplify the logic
+  void update_status(std::string _file_name, CACHESTATUS _status) {
+    Mutex::Locker locker(m_lock);
+
+    Entry* entry;
+    auto entry_it = m_oid_to_entry.find(_file_name);
+
+    // just check.
+    if(_status == PROMOTING) {
+      assert(m_oid_to_entry.find(_file_name) == m_oid_to_entry.end());
+    }
+
+    // miss this object.
+    if(entry_it == m_oid_to_entry.end() && _status == PROMOTING) {
+      entry = reinterpret_cast<Entry*>(m_free_lru.lru_get_next_expire());
+      if(entry == nullptr) {
+        assert(0); // namely evict thread have some problems.
+      }
+
+      entry->status = PROMOTING;
+
+      m_oid_to_entry[_file_name] = entry;
+      m_free_lru.lru_remove(entry);
+      m_handing_lru.lru_insert_top(entry);
+
+      return;
+    }
+
+    assert(entry_it != m_oid_to_entry.end());
+
+    entry = entry_it->second;
+
+    // promoting action have been finished, so update it.
+    if(entry->status == PROMOTING && _status== PROMOTED) {
+      m_handing_lru.lru_remove(entry);
+      m_promoted_lru.lru_insert_top(entry);
+      entry->status = PROMOTED;
+      return;
+    }
+
+    // will delete this cache file
+    if(entry->status == PROMOTED && _status == EVICTING) {
+      m_promoted_lru.lru_remove(entry);
+      m_handing_lru.lru_insert_top(entry);
+      entry->status = EVICTING;
+      return;
+    }
+
+
+    if(_status == EVICTED) {
+      m_oid_to_entry.erase(entry_it);
+      m_handing_lru.lru_remove(entry);
+      m_free_lru.lru_insert_bot(entry);
+      return;
+    }
+
+    assert(0);
+  }
+
+  // get entry status
+  CACHESTATUS get_status(std::string _file_name) {
+    Mutex::Locker locker(m_lock);
+    auto entry_it = m_oid_to_entry.find(_file_name);
+    if(entry_it == m_oid_to_entry.end()) {
+      return NONE;
+    }
+
+    return entry_it->second->status;
+  }
+
+
+private:
+
+  class Entry : public LRUObject {
+    public:
+      CACHESTATUS status;
+      Entry() : status(NONE){}
+      std::string cache_file_name;
+      void encode(bufferlist &bl){}
+      void decode(bufferlist::iterator &it){}
+  };
+
+  std::unordered_map<std::string, Entry*> m_oid_to_entry;
+
+  LRU m_free_lru;
+  LRU m_handing_lru; // include promoting status or evicting status
+  LRU m_promoted_lru; // include promoted, using status.
+
+  mutable Mutex m_lock;
+
+  float m_level;
+  uint64_t m_entry_count;
+
+};
+
+#endif
-- 
2.7.4