path: root/src/ceph/0001-librbd-shared-persistent-read-only-rbd-cache.patch

From b7b81562c76011abe05930330915a5ba423964e4 Mon Sep 17 00:00:00 2001
From: Yuan Zhou <yuan.zhou@intel.com>
Date: Thu, 19 Apr 2018 22:54:36 +0800
Subject: [PATCH 01/10] librbd: shared persistent read-only rbd cache

This patch introduces introduces RBD shared persistent RO cache which
can provide client-side sharing cache for rbd clone/snapshot case.

The key componenets are:

- RBD cache daemon runs on each compute node to control the shared cache state

- Read-only blocks from parent image(s) are cached in a shared area on
  compute node(s)

- Object level dispatcher inside librbd that can do RPC with cache daemon to
  lookup the cache

- Reads are served from the shared cache until the first COW request

- Policy to control promotion/evication of the shared cache

The general IO flow is:

0) Parent image would register themselfs when initializing

1) When read request on cloned image flows to parent image, it will check with
   the cache daemon if the rarget object is ready

2) Cache daemon receives the lookup request:
   a) if the target object is promoted, daemon will ack with "read_from_cache"
   b) if it is not promoted, daemon will check the policy whether to promote:
      - if yes, daemon will do the promiton then ack with "read_from_cache"
      - if no, daemon will ack with "read_from_rados"

3) the read reqeust contines to do read from cache/rados based on the ack

Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
---
 src/common/options.cc                              |   8 ++
 src/librbd/CMakeLists.txt                          |   4 +-
 src/librbd/ImageCtx.cc                             |   5 +-
 src/librbd/ImageCtx.h                              |   3 +
 src/librbd/cache/SharedPersistentObjectCacher.cc   |  61 ++++++++
 src/librbd/cache/SharedPersistentObjectCacher.h    |  45 ++++++
 .../SharedPersistentObjectCacherObjectDispatch.cc  | 154 +++++++++++++++++++++
 .../SharedPersistentObjectCacherObjectDispatch.h   | 127 +++++++++++++++++
 src/librbd/image/OpenRequest.cc                    |  12 +-
 src/librbd/io/Types.h                              |   1 +
 src/os/CacheStore/SyncFile.cc                      | 110 +++++++++++++++
 src/os/CacheStore/SyncFile.h                       |  74 ++++++++++
 src/test/librbd/test_mirroring.cc                  |   1 +
 src/test/rbd_mirror/test_ImageReplayer.cc          |   2 +
 src/test/rbd_mirror/test_fixture.cc                |   1 +
 src/tools/CMakeLists.txt                           |   1 +
 src/tools/rbd_cache/CMakeLists.txt                 |   9 ++
 src/tools/rbd_cache/CacheController.cc             | 105 ++++++++++++++
 src/tools/rbd_cache/CacheController.hpp            |  49 +++++++
 src/tools/rbd_cache/CacheControllerSocket.hpp      | 125 +++++++++++++++++
 .../rbd_cache/CacheControllerSocketClient.hpp      | 131 ++++++++++++++++++
 src/tools/rbd_cache/CacheControllerSocketCommon.h  |  43 ++++++
 src/tools/rbd_cache/ObjectCacheStore.cc            | 147 ++++++++++++++++++++
 src/tools/rbd_cache/ObjectCacheStore.h             |  65 +++++++++
 src/tools/rbd_cache/main.cc                        |  85 ++++++++++++
 25 files changed, 1365 insertions(+), 3 deletions(-)
 create mode 100644 src/librbd/cache/SharedPersistentObjectCacher.cc
 create mode 100644 src/librbd/cache/SharedPersistentObjectCacher.h
 create mode 100644 src/librbd/cache/SharedPersistentObjectCacherObjectDispatch.cc
 create mode 100644 src/librbd/cache/SharedPersistentObjectCacherObjectDispatch.h
 create mode 100644 src/os/CacheStore/SyncFile.cc
 create mode 100644 src/os/CacheStore/SyncFile.h
 create mode 100644 src/tools/rbd_cache/CMakeLists.txt
 create mode 100644 src/tools/rbd_cache/CacheController.cc
 create mode 100644 src/tools/rbd_cache/CacheController.hpp
 create mode 100644 src/tools/rbd_cache/CacheControllerSocket.hpp
 create mode 100644 src/tools/rbd_cache/CacheControllerSocketClient.hpp
 create mode 100644 src/tools/rbd_cache/CacheControllerSocketCommon.h
 create mode 100644 src/tools/rbd_cache/ObjectCacheStore.cc
 create mode 100644 src/tools/rbd_cache/ObjectCacheStore.h
 create mode 100644 src/tools/rbd_cache/main.cc

diff --git a/src/common/options.cc b/src/common/options.cc
index c5afe4c..7839a31 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -6357,6 +6357,14 @@ static std::vector<Option> get_rbd_options() {
     .set_default(60)
     .set_description("time in seconds for detecting a hung thread"),
 
+    Option("rbd_shared_cache_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .set_description("whether to enable shared ssd caching"),
+
+    Option("rbd_shared_cache_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
+    .set_default("/tmp")
+    .set_description("shared ssd caching data dir"),
+
     Option("rbd_non_blocking_aio", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(true)
     .set_description("process AIO ops from a dispatch thread to prevent blocking"),
diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt
index b9c08d4..92539a8 100644
--- a/src/librbd/CMakeLists.txt
+++ b/src/librbd/CMakeLists.txt
@@ -32,7 +32,8 @@ set(librbd_internal_srcs
   api/Snapshot.cc
   cache/ImageWriteback.cc
   cache/ObjectCacherObjectDispatch.cc
-  cache/PassthroughImageCache.cc
+  cache/SharedPersistentObjectCacherObjectDispatch.cc
+  cache/SharedPersistentObjectCacher.cc
   deep_copy/ImageCopyRequest.cc
   deep_copy/MetadataCopyRequest.cc
   deep_copy/ObjectCopyRequest.cc
@@ -123,6 +124,7 @@ set(librbd_internal_srcs
   trash/MoveRequest.cc
   watcher/Notifier.cc
   watcher/RewatchRequest.cc
+  ${CMAKE_SOURCE_DIR}/src/os/CacheStore/SyncFile.cc
   ${CMAKE_SOURCE_DIR}/src/common/ContextCompletion.cc)
 
 add_library(rbd_api STATIC librbd.cc)
diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index 48f98b1..349156b 100644
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -776,7 +776,8 @@ public:
 	"rbd_qos_read_iops_limit", false)(
 	"rbd_qos_write_iops_limit", false)(
 	"rbd_qos_read_bps_limit", false)(
-	"rbd_qos_write_bps_limit", false);
+	"rbd_qos_write_bps_limit", false)(
+        "rbd_shared_cache_enabled", false);
 
     ConfigProxy local_config_t{false};
     std::map<std::string, bufferlist> res;
@@ -844,6 +845,8 @@ public:
     ASSIGN_OPTION(qos_write_iops_limit, uint64_t);
     ASSIGN_OPTION(qos_read_bps_limit, uint64_t);
     ASSIGN_OPTION(qos_write_bps_limit, uint64_t);
+    ASSIGN_OPTION(shared_cache_enabled, bool);
+    ASSIGN_OPTION(shared_cache_path, std::string);
 
     if (thread_safe) {
       ASSIGN_OPTION(journal_pool, std::string);
diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index d197c24..f661c09 100644
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -204,6 +204,9 @@ namespace librbd {
     uint64_t qos_read_bps_limit;
     uint64_t qos_write_bps_limit;
 
+    bool shared_cache_enabled;
+    std::string shared_cache_path;
+
     LibrbdAdminSocketHook *asok_hook;
 
     exclusive_lock::Policy *exclusive_lock_policy = nullptr;
diff --git a/src/librbd/cache/SharedPersistentObjectCacher.cc b/src/librbd/cache/SharedPersistentObjectCacher.cc
new file mode 100644
index 0000000..a849260
--- /dev/null
+++ b/src/librbd/cache/SharedPersistentObjectCacher.cc
@@ -0,0 +1,61 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/cache/SharedPersistentObjectCacher.h"
+#include "include/buffer.h"
+#include "common/dout.h"
+#include "librbd/ImageCtx.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::cache::SharedPersistentObjectCacher: " << this \
+                           << " " <<  __func__ << ": "
+
+namespace librbd {
+namespace cache {
+
+template <typename I>
+SharedPersistentObjectCacher<I>::SharedPersistentObjectCacher(I *image_ctx, std::string cache_path)
+  : m_image_ctx(image_ctx), m_cache_path(cache_path),
+    m_file_map_lock("librbd::cache::SharedObjectCacher::filemaplock") {
+  auto *cct = m_image_ctx->cct;
+
+}
+
+template <typename I>
+SharedPersistentObjectCacher<I>::~SharedPersistentObjectCacher() {
+  for(auto &it: file_map) {
+    if(it.second) {
+      delete it.second;
+    }
+  }
+}
+
+template <typename I>
+int SharedPersistentObjectCacher<I>::read_object(std::string oid, ceph::bufferlist* read_data, uint64_t offset, uint64_t length, Context *on_finish) {
+
+  auto *cct = m_image_ctx->cct;
+  ldout(cct, 20) << "object: " << oid << dendl;
+
+  std::string cache_file_name = m_image_ctx->data_ctx.get_pool_name() + oid;
+
+  //TODO(): make a cache for cachefile fd
+  os::CacheStore::SyncFile* target_cache_file = new os::CacheStore::SyncFile(cct, cache_file_name);
+  target_cache_file->open();
+
+  int ret = target_cache_file->read_object_from_file(read_data, offset, length);
+  if (ret < 0) {
+    ldout(cct, 5) << "read from file return error: " << ret 
+                  << "file name= " << cache_file_name
+                  << dendl;
+  }
+
+  delete target_cache_file;
+  return ret;
+}
+
+
+} // namespace cache
+} // namespace librbd
+
+template class librbd::cache::SharedPersistentObjectCacher<librbd::ImageCtx>;
diff --git a/src/librbd/cache/SharedPersistentObjectCacher.h b/src/librbd/cache/SharedPersistentObjectCacher.h
new file mode 100644
index 0000000..d108a05
--- /dev/null
+++ b/src/librbd/cache/SharedPersistentObjectCacher.h
@@ -0,0 +1,45 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_CACHE_SHARED_PERSISTENT_OBJECT_CACHER
+#define CEPH_LIBRBD_CACHE_SHARED_PERSISTENT_OBJECT_CACHER
+
+#include "include/buffer_fwd.h"
+#include "include/int_types.h"
+#include "os/CacheStore/SyncFile.h"
+#include "common/Mutex.h"
+#include <vector>
+#include <map>
+
+struct Context;
+
+namespace librbd {
+
+struct ImageCtx;
+
+namespace cache {
+
+template <typename ImageCtxT>
+class SharedPersistentObjectCacher {
+public:
+
+  SharedPersistentObjectCacher(ImageCtxT *image_ctx, std::string cache_path);
+  ~SharedPersistentObjectCacher();
+
+  int read_object(std::string oid, ceph::bufferlist* read_data,
+		  uint64_t offset, uint64_t length, Context *on_finish);
+
+private:
+  ImageCtxT *m_image_ctx;
+  std::map<std::string,  os::CacheStore::SyncFile*> file_map;
+  Mutex m_file_map_lock;
+  std::string m_cache_path;
+
+};
+
+} // namespace cache
+} // namespace librbd
+
+extern template class librbd::cache::SharedPersistentObjectCacher<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_CACHE_FILE_IMAGE_STORE
diff --git a/src/librbd/cache/SharedPersistentObjectCacherObjectDispatch.cc b/src/librbd/cache/SharedPersistentObjectCacherObjectDispatch.cc
new file mode 100644
index 0000000..90d886c
--- /dev/null
+++ b/src/librbd/cache/SharedPersistentObjectCacherObjectDispatch.cc
@@ -0,0 +1,154 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/cache/SharedPersistentObjectCacherObjectDispatch.h"
+#include "common/WorkQueue.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/Journal.h"
+#include "librbd/Utils.h"
+#include "librbd/LibrbdWriteback.h"
+#include "librbd/io/ObjectDispatchSpec.h"
+#include "librbd/io/ObjectDispatcher.h"
+#include "librbd/io/Utils.h"
+#include "osd/osd_types.h"
+#include "osdc/WritebackHandler.h"
+#include <vector>
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::cache::SharedPersistentObjectCacherObjectDispatch: " \
+                           << this << " " << __func__ << ": "
+
+namespace librbd {
+namespace cache {
+
+template <typename I>
+SharedPersistentObjectCacherObjectDispatch<I>::SharedPersistentObjectCacherObjectDispatch(
+    I* image_ctx) : m_image_ctx(image_ctx) {
+}
+
+template <typename I>
+SharedPersistentObjectCacherObjectDispatch<I>::~SharedPersistentObjectCacherObjectDispatch() {
+  if (m_object_store) {
+    delete m_object_store;
+  }
+
+  if (m_cache_client) {
+    delete m_cache_client;
+  }
+}
+
+template <typename I>
+void SharedPersistentObjectCacherObjectDispatch<I>::init() {
+  auto cct = m_image_ctx->cct;
+  ldout(cct, 5) << dendl;
+
+  if (m_image_ctx->parent != nullptr) {
+    //TODO(): should we cover multi-leveled clone?
+    ldout(cct, 5) << "child image: skipping SRO cache client" << dendl;
+    return;
+  }
+
+  ldout(cct, 20) << "parent image: setup SRO cache client = " << dendl;
+
+  std::string controller_path = "/tmp/rbd_shared_readonly_cache_demo";
+  m_cache_client = new CacheClient(io_service, controller_path.c_str(),
+    ([&](std::string s){client_handle_request(s);}));
+
+  int ret = m_cache_client->connect();
+  if (ret < 0) {
+    ldout(cct, 5) << "SRO cache client fail to connect with local controller: "
+                  << "please start rbd-cache daemon"
+		  << dendl;
+  } else {
+    ldout(cct, 5) << "SRO cache client to register volume on rbd-cache daemon: "
+                   << "name = " << m_image_ctx->id 
+                   << dendl;
+
+    ret = m_cache_client->register_volume(m_image_ctx->data_ctx.get_pool_name(),
+                                    m_image_ctx->id, m_image_ctx->size);
+
+    if (ret >= 0) {
+      // add ourself to the IO object dispatcher chain
+      m_image_ctx->io_object_dispatcher->register_object_dispatch(this);
+    }
+  }
+}
+
+template <typename I>
+bool SharedPersistentObjectCacherObjectDispatch<I>::read(
+    const std::string &oid, uint64_t object_no, uint64_t object_off,
+    uint64_t object_len, librados::snap_t snap_id, int op_flags,
+    const ZTracer::Trace &parent_trace, ceph::bufferlist* read_data,
+    io::ExtentMap* extent_map, int* object_dispatch_flags,
+    io::DispatchResult* dispatch_result, Context** on_finish,
+    Context* on_dispatched) {
+  // IO chained in reverse order
+  auto cct = m_image_ctx->cct;
+  ldout(cct, 20) << "object_no=" << object_no << " " << object_off << "~"
+                 << object_len << dendl;
+
+  // ensure we aren't holding the cache lock post-read
+  on_dispatched = util::create_async_context_callback(*m_image_ctx,
+                                                      on_dispatched);
+
+  if (m_cache_client && m_cache_client->connected && m_object_store) {
+    bool exists;
+    m_cache_client->lookup_object(m_image_ctx->data_ctx.get_pool_name(),
+      m_image_ctx->id, oid, &exists);
+
+    // try to read from parent image
+    ldout(cct, 20) << "SRO cache object exists:" << exists << dendl;
+    if (exists) {
+      int r = m_object_store->read_object(oid, read_data, object_off, object_len, on_dispatched);
+      if (r != 0) {
+        *dispatch_result = io::DISPATCH_RESULT_COMPLETE;
+	on_dispatched->complete(r);
+        return true;
+      }
+    }
+  }
+
+  ldout(cct, 20) << "Continue read from RADOS" << dendl;
+  *dispatch_result = io::DISPATCH_RESULT_CONTINUE;
+  on_dispatched->complete(0);
+  return true;
+}
+
+template <typename I>
+void SharedPersistentObjectCacherObjectDispatch<I>::client_handle_request(std::string msg) {
+  auto cct = m_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  rbdsc_req_type_t *io_ctx = (rbdsc_req_type_t*)(msg.c_str());
+
+  switch (io_ctx->type) {
+    case RBDSC_REGISTER_REPLY: {
+      // open cache handler for volume        
+      ldout(cct, 20) << "SRO cache client open cache handler" << dendl;
+      m_object_store = new SharedPersistentObjectCacher<I>(m_image_ctx, m_image_ctx->shared_cache_path);
+
+      break;
+    }
+    case RBDSC_READ_REPLY: {
+      ldout(cct, 20) << "SRO cache client start to read cache" << dendl;
+      //TODO(): should call read here
+
+      break;
+    }
+    case RBDSC_READ_RADOS: {
+      ldout(cct, 20) << "SRO cache client start to read rados" << dendl;
+      //TODO(): should call read here
+
+      break;
+    }
+    default: ldout(cct, 20) << "nothing" << dendl;
+      break;
+    
+  }
+}
+
+} // namespace cache
+} // namespace librbd
+
+template class librbd::cache::SharedPersistentObjectCacherObjectDispatch<librbd::ImageCtx>;
diff --git a/src/librbd/cache/SharedPersistentObjectCacherObjectDispatch.h b/src/librbd/cache/SharedPersistentObjectCacherObjectDispatch.h
new file mode 100644
index 0000000..1ede804
--- /dev/null
+++ b/src/librbd/cache/SharedPersistentObjectCacherObjectDispatch.h
@@ -0,0 +1,127 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_CACHE_SHARED_PERSISTENT_OBJECT_CACHER_OBJECT_DISPATCH_H
+#define CEPH_LIBRBD_CACHE_SHARED_PERSISTENT_OBJECT_CACHER_OBJECT_DISPATCH_H
+
+#include "librbd/io/ObjectDispatchInterface.h"
+#include "common/Mutex.h"
+#include "osdc/ObjectCacher.h"
+#include "tools/rbd_cache/CacheControllerSocketClient.hpp"
+#include "SharedPersistentObjectCacher.h"
+
+struct WritebackHandler;
+
+namespace librbd {
+
+class ImageCtx;
+
+namespace cache {
+
+/**
+ * Facade around the OSDC object cacher to make it align with
+ * the object dispatcher interface
+ */
+template <typename ImageCtxT = ImageCtx>
+class SharedPersistentObjectCacherObjectDispatch : public io::ObjectDispatchInterface {
+public:
+  static SharedPersistentObjectCacherObjectDispatch* create(ImageCtxT* image_ctx) {
+    return new SharedPersistentObjectCacherObjectDispatch(image_ctx);
+  }
+
+  SharedPersistentObjectCacherObjectDispatch(ImageCtxT* image_ctx);
+  ~SharedPersistentObjectCacherObjectDispatch() override;
+
+  io::ObjectDispatchLayer get_object_dispatch_layer() const override {
+    return io::OBJECT_DISPATCH_LAYER_SHARED_PERSISTENT_CACHE;
+  }
+
+  void init();
+  void shut_down(Context* on_finish) {
+    m_image_ctx->op_work_queue->queue(on_finish, 0);
+  }
+
+  bool read(
+      const std::string &oid, uint64_t object_no, uint64_t object_off,
+      uint64_t object_len, librados::snap_t snap_id, int op_flags,
+      const ZTracer::Trace &parent_trace, ceph::bufferlist* read_data,
+      io::ExtentMap* extent_map, int* object_dispatch_flags,
+      io::DispatchResult* dispatch_result, Context** on_finish,
+      Context* on_dispatched) override;
+
+  bool discard(
+      const std::string &oid, uint64_t object_no, uint64_t object_off,
+      uint64_t object_len, const ::SnapContext &snapc, int discard_flags,
+      const ZTracer::Trace &parent_trace, int* object_dispatch_flags,
+      uint64_t* journal_tid, io::DispatchResult* dispatch_result,
+      Context** on_finish, Context* on_dispatched) {
+    return false;
+  }
+
+  bool write(
+      const std::string &oid, uint64_t object_no, uint64_t object_off,
+      ceph::bufferlist&& data, const ::SnapContext &snapc, int op_flags,
+      const ZTracer::Trace &parent_trace, int* object_dispatch_flags,
+      uint64_t* journal_tid, io::DispatchResult* dispatch_result,
+      Context** on_finish, Context* on_dispatched) {
+    return false;
+  }
+
+  bool write_same(
+      const std::string &oid, uint64_t object_no, uint64_t object_off,
+      uint64_t object_len, io::Extents&& buffer_extents,
+      ceph::bufferlist&& data, const ::SnapContext &snapc, int op_flags,
+      const ZTracer::Trace &parent_trace, int* object_dispatch_flags,
+      uint64_t* journal_tid, io::DispatchResult* dispatch_result,
+      Context** on_finish, Context* on_dispatched) {
+    return false;
+  }
+
+  bool compare_and_write(
+      const std::string &oid, uint64_t object_no, uint64_t object_off,
+      ceph::bufferlist&& cmp_data, ceph::bufferlist&& write_data,
+      const ::SnapContext &snapc, int op_flags,
+      const ZTracer::Trace &parent_trace, uint64_t* mismatch_offset,
+      int* object_dispatch_flags, uint64_t* journal_tid,
+      io::DispatchResult* dispatch_result, Context** on_finish,
+      Context* on_dispatched) {
+    return false;
+  }
+
+  bool flush(
+      io::FlushSource flush_source, const ZTracer::Trace &parent_trace,
+      io::DispatchResult* dispatch_result, Context** on_finish,
+      Context* on_dispatched) {
+    return false;
+  }
+
+  bool invalidate_cache(Context* on_finish) {
+    return false;
+  }
+
+  bool reset_existence_cache(Context* on_finish) {
+    return false;
+  }
+
+  void extent_overwritten(
+      uint64_t object_no, uint64_t object_off, uint64_t object_len,
+      uint64_t journal_tid, uint64_t new_journal_tid) {
+  }
+
+  SharedPersistentObjectCacher<ImageCtxT> *m_object_store = nullptr;
+
+private:
+
+  ImageCtxT* m_image_ctx;
+
+  void client_handle_request(std::string msg);
+  CacheClient *m_cache_client = nullptr;
+  boost::asio::io_service io_service;
+};
+
+} // namespace cache
+} // namespace librbd
+
+extern template class librbd::cache::SharedPersistentObjectCacherObjectDispatch<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_CACHE_OBJECT_CACHER_OBJECT_DISPATCH_H
diff --git a/src/librbd/image/OpenRequest.cc b/src/librbd/image/OpenRequest.cc
index ae18739..30a7b66 100644
--- a/src/librbd/image/OpenRequest.cc
+++ b/src/librbd/image/OpenRequest.cc
@@ -8,6 +8,7 @@
 #include "librbd/ImageCtx.h"
 #include "librbd/Utils.h"
 #include "librbd/cache/ObjectCacherObjectDispatch.h"
+#include "librbd/cache/SharedPersistentObjectCacherObjectDispatch.cc"
 #include "librbd/image/CloseRequest.h"
 #include "librbd/image/RefreshRequest.h"
 #include "librbd/image/SetSnapRequest.h"
@@ -448,12 +449,21 @@ Context *OpenRequest<I>::handle_refresh(int *result) {
 
 template <typename I>
 Context *OpenRequest<I>::send_init_cache(int *result) {
+
+  CephContext *cct = m_image_ctx->cct;
   // cache is disabled or parent image context
   if (!m_image_ctx->cache || m_image_ctx->child != nullptr) {
+
+    // enable Shared Read-only cache for parent image
+    if (m_image_ctx->child != nullptr && m_image_ctx->shared_cache_enabled ) {
+      ldout(cct, 10) << this << " " << "setting up parent cache"<< dendl;
+      auto sro_cache = cache::SharedPersistentObjectCacherObjectDispatch<I>::create(m_image_ctx);
+      sro_cache->init();
+    }
+
     return send_register_watch(result);
   }
 
-  CephContext *cct = m_image_ctx->cct;
   ldout(cct, 10) << this << " " << __func__ << dendl;
 
   auto cache = cache::ObjectCacherObjectDispatch<I>::create(m_image_ctx);
diff --git a/src/librbd/io/Types.h b/src/librbd/io/Types.h
index 7e09c90..ef3049f 100644
--- a/src/librbd/io/Types.h
+++ b/src/librbd/io/Types.h
@@ -59,6 +59,7 @@ enum DispatchResult {
 enum ObjectDispatchLayer {
   OBJECT_DISPATCH_LAYER_NONE = 0,
   OBJECT_DISPATCH_LAYER_CACHE,
+  OBJECT_DISPATCH_LAYER_SHARED_PERSISTENT_CACHE,
   OBJECT_DISPATCH_LAYER_JOURNAL,
   OBJECT_DISPATCH_LAYER_CORE,
   OBJECT_DISPATCH_LAYER_LAST
diff --git a/src/os/CacheStore/SyncFile.cc b/src/os/CacheStore/SyncFile.cc
new file mode 100644
index 0000000..5352bde
--- /dev/null
+++ b/src/os/CacheStore/SyncFile.cc
@@ -0,0 +1,110 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "os/CacheStore/SyncFile.h"
+#include "include/Context.h"
+#include "common/dout.h"
+#include "common/WorkQueue.h"
+#include "librbd/ImageCtx.h"
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <aio.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <utility>
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::file::SyncFile: " << this << " " \
+                           <<  __func__ << ": "
+
+namespace os {
+namespace CacheStore {
+
+SyncFile::SyncFile(CephContext *cct, const std::string &name)
+  : cct(cct)
+{
+  m_name = cct->_conf->get_val<std::string>("rbd_shared_cache_path") + "/rbd_cache." + name;
+  ldout(cct, 20) << "file path=" << m_name << dendl;
+}
+
+SyncFile::~SyncFile() 
+{
+  // TODO force proper cleanup
+  if (m_fd != -1) {
+    ::close(m_fd);
+  }
+}
+
+void SyncFile::open(Context *on_finish) 
+{
+  while (true) {
+    m_fd = ::open(m_name.c_str(), O_CREAT | O_DIRECT | O_NOATIME | O_RDWR | O_SYNC,
+                  S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
+    if (m_fd == -1) {
+      int r = -errno;
+      if (r == -EINTR) {
+        continue;
+      }
+      on_finish->complete(r);
+      return;
+    }
+    break;
+  }
+
+  on_finish->complete(0);
+}
+
+void SyncFile::open() 
+{
+  while (true) 
+  {
+    m_fd = ::open(m_name.c_str(), O_CREAT | O_NOATIME | O_RDWR | O_SYNC,
+                  S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
+    if (m_fd == -1) 
+    {
+      int r = -errno;
+      if (r == -EINTR) {
+        continue;
+      }
+      return;
+    }
+    break;
+  }
+}
+
+int SyncFile::write_object_to_file(ceph::bufferlist read_buf, uint64_t object_len) {
+
+  ldout(cct, 20) << "cache file name:" << m_name
+                 << ", length:" << object_len <<  dendl;
+
+  // TODO(): aio
+  int ret = pwrite(m_fd, read_buf.c_str(), object_len, 0); 
+  if(ret < 0) {
+    lderr(cct)<<"write file fail:" << std::strerror(errno) << dendl;
+    return ret;
+  }
+
+  return ret;
+}
+
+int SyncFile::read_object_from_file(ceph::bufferlist* read_buf, uint64_t object_off, uint64_t object_len) {
+
+  ldout(cct, 20) << "offset:" << object_off
+                 << ", length:" << object_len <<  dendl;
+
+  bufferptr buf(object_len);
+
+  // TODO(): aio
+  int ret = pread(m_fd, buf.c_str(), object_len, object_off); 
+  if(ret < 0) {
+    lderr(cct)<<"read file fail:" << std::strerror(errno) << dendl;
+    return ret;
+  }
+  read_buf->append(std::move(buf));
+
+  return ret;
+}
+
+} // namespace CacheStore
+} // namespace os
diff --git a/src/os/CacheStore/SyncFile.h b/src/os/CacheStore/SyncFile.h
new file mode 100644
index 0000000..81602ce
--- /dev/null
+++ b/src/os/CacheStore/SyncFile.h
@@ -0,0 +1,74 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBOS_CACHE_STORE_SYNC_FILE
+#define CEPH_LIBOS_CACHE_STORE_SYNC_FILE
+
+#include "include/buffer_fwd.h"
+#include <sys/mman.h>
+#include <string>
+
+struct Context;
+struct ContextWQ;
+class CephContext;
+
+namespace os {
+
+namespace CacheStore {
+
+class SyncFile {
+public:
+  SyncFile(CephContext *cct, const std::string &name);
+  ~SyncFile();
+
+  // TODO use IO queue instead of individual commands so operations can be
+  // submitted in batch
+
+  // TODO use scatter/gather API
+
+  void open(Context *on_finish);
+
+  // ##
+  void open();
+  bool try_open();
+  void close(Context *on_finish);
+  void remove(Context *on_finish);
+
+  void read(uint64_t offset, uint64_t length, ceph::bufferlist *bl, Context *on_finish);
+
+  void write(uint64_t offset, ceph::bufferlist &&bl, bool fdatasync, Context *on_finish);
+
+  void discard(uint64_t offset, uint64_t length, bool fdatasync, Context *on_finish);
+
+  void truncate(uint64_t length, bool fdatasync, Context *on_finish);
+
+  void fsync(Context *on_finish);
+
+  void fdatasync(Context *on_finish);
+
+  uint64_t filesize();
+
+  int load(void** dest, uint64_t filesize);
+
+  int remove();
+
+  // ##
+  int write_object_to_file(ceph::bufferlist read_buf, uint64_t object_len);
+  int read_object_from_file(ceph::bufferlist* read_buf, uint64_t object_off, uint64_t object_len);
+
+private:
+  CephContext *cct;
+  std::string m_name;
+  int m_fd = -1;
+
+  int write(uint64_t offset, const ceph::bufferlist &bl, bool fdatasync);
+  int read(uint64_t offset, uint64_t length, ceph::bufferlist *bl);
+  int discard(uint64_t offset, uint64_t length, bool fdatasync);
+  int truncate(uint64_t length, bool fdatasync);
+  int fdatasync();
+};
+
+} // namespace CacheStore
+} // namespace os
+
+#endif // CEPH_LIBOS_CACHE_STORE_SYNC_FILE
diff --git a/src/test/librbd/test_mirroring.cc b/src/test/librbd/test_mirroring.cc
index b4fdeae..d7d1aa6 100644
--- a/src/test/librbd/test_mirroring.cc
+++ b/src/test/librbd/test_mirroring.cc
@@ -47,6 +47,7 @@ public:
 
   void SetUp() override {
     ASSERT_EQ(0, _rados.ioctx_create(_pool_name.c_str(), m_ioctx));
+    ASSERT_EQ(0, _rados.conf_set("rbd_shared_cache_enabled", "false"));
   }
 
   std::string image_name = "mirrorimg1";
diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc
index 8a95a65..b5598bd 100644
--- a/src/test/rbd_mirror/test_ImageReplayer.cc
+++ b/src/test/rbd_mirror/test_ImageReplayer.cc
@@ -90,6 +90,7 @@ public:
     EXPECT_EQ("", connect_cluster_pp(*m_local_cluster.get()));
     EXPECT_EQ(0, m_local_cluster->conf_set("rbd_cache", "false"));
     EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirror_journal_poll_age", "1"));
+    EXPECT_EQ(0, m_local_cluster->conf_set("rbd_shared_cache_enabled", "false"));
 
     m_local_pool_name = get_temp_pool_name();
     EXPECT_EQ(0, m_local_cluster->pool_create(m_local_pool_name.c_str()));
@@ -99,6 +100,7 @@ public:
 
     EXPECT_EQ("", connect_cluster_pp(m_remote_cluster));
     EXPECT_EQ(0, m_remote_cluster.conf_set("rbd_cache", "false"));
+    EXPECT_EQ(0, m_remote_cluster.conf_set("rbd_shared_cache_enabled", "false"));
 
     m_remote_pool_name = get_temp_pool_name();
     EXPECT_EQ(0, m_remote_cluster.pool_create(m_remote_pool_name.c_str()));
diff --git a/src/test/rbd_mirror/test_fixture.cc b/src/test/rbd_mirror/test_fixture.cc
index b2a51ca..9e77098 100644
--- a/src/test/rbd_mirror/test_fixture.cc
+++ b/src/test/rbd_mirror/test_fixture.cc
@@ -27,6 +27,7 @@ void TestFixture::SetUpTestCase() {
   _rados = std::shared_ptr<librados::Rados>(new librados::Rados());
   ASSERT_EQ("", connect_cluster_pp(*_rados.get()));
   ASSERT_EQ(0, _rados->conf_set("rbd_cache", "false"));
+  ASSERT_EQ(0, _rados->conf_set("rbd_shared_cache_enabled", "false"));
 
   _local_pool_name = get_temp_pool_name("test-rbd-mirror-");
   ASSERT_EQ(0, _rados->pool_create(_local_pool_name.c_str()));
diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt
index 3789e3c..72ab342 100644
--- a/src/tools/CMakeLists.txt
+++ b/src/tools/CMakeLists.txt
@@ -99,6 +99,7 @@ endif(WITH_CEPHFS)
 if(WITH_RBD)
   add_subdirectory(rbd)
   add_subdirectory(rbd_mirror)
+  add_subdirectory(rbd_cache)
   if(LINUX)
     add_subdirectory(rbd_nbd)
   endif()
diff --git a/src/tools/rbd_cache/CMakeLists.txt b/src/tools/rbd_cache/CMakeLists.txt
new file mode 100644
index 0000000..08eae60
--- /dev/null
+++ b/src/tools/rbd_cache/CMakeLists.txt
@@ -0,0 +1,9 @@
+add_executable(rbd-cache
+  ${CMAKE_SOURCE_DIR}/src/os/CacheStore/SyncFile.cc
+  ObjectCacheStore.cc
+  CacheController.cc
+  main.cc)
+target_link_libraries(rbd-cache
+  librados
+  global)
+install(TARGETS rbd-cache DESTINATION bin)
diff --git a/src/tools/rbd_cache/CacheController.cc b/src/tools/rbd_cache/CacheController.cc
new file mode 100644
index 0000000..c914358
--- /dev/null
+++ b/src/tools/rbd_cache/CacheController.cc
@@ -0,0 +1,105 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "CacheController.hpp"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_cache
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::cache::CacheController: " << this << " " \
+                           << __func__ << ": "
+
+
+class ThreadPoolSingleton : public ThreadPool {
+public:
+  ContextWQ *op_work_queue;
+
+  explicit ThreadPoolSingleton(CephContext *cct)
+    : ThreadPool(cct, "librbd::cache::thread_pool", "tp_librbd_cache", 32,
+                 "pcache_threads"),
+      op_work_queue(new ContextWQ("librbd::pcache_op_work_queue",
+                    cct->_conf->get_val<int64_t>("rbd_op_thread_timeout"),
+                    this)) {
+    start();
+  }
+  ~ThreadPoolSingleton() override {
+    op_work_queue->drain();
+    delete op_work_queue;
+
+    stop();
+  }
+};
+
+
+CacheController::CacheController(CephContext *cct, const std::vector<const char*> &args):
+  m_args(args), m_cct(cct) {
+
+}
+
+CacheController::~CacheController() {
+
+}
+
+int CacheController::init() {
+  ThreadPoolSingleton* thread_pool_singleton = &m_cct->lookup_or_create_singleton_object<ThreadPoolSingleton>(
+    "rbd::cache::thread_pool", false, m_cct);
+  pcache_op_work_queue = thread_pool_singleton->op_work_queue;
+
+  m_object_cache_store = new ObjectCacheStore(m_cct, pcache_op_work_queue);
+  int r = m_object_cache_store->init(false);
+  if (r < 0) {
+    //derr << "init error\n" << dendl;
+  }
+  return r;
+}
+
+int CacheController::shutdown() {
+  int r = m_object_cache_store->shutdown();
+  return r;
+}
+
+void CacheController::handle_signal(int signum){}
+
+void CacheController::run() {
+  try {
+    //TODO(): use new socket path
+    std::string controller_path = "/tmp/rbd_shared_readonly_cache_demo";
+    std::remove(controller_path.c_str()); 
+    
+    m_cache_server = new CacheServer(io_service, controller_path,
+      ([&](uint64_t p, std::string s){handle_request(p, s);}));
+    io_service.run();
+  } catch (std::exception& e) {
+    std::cerr << "Exception: " << e.what() << "\n";
+  }
+}
+
+void CacheController::handle_request(uint64_t sesstion_id, std::string msg){
+  rbdsc_req_type_t *io_ctx = (rbdsc_req_type_t*)(msg.c_str());
+
+  int ret = 0;
+  
+  switch (io_ctx->type) {
+    case RBDSC_REGISTER: {
+      // init cache layout for volume        
+      m_object_cache_store->init_cache(io_ctx->vol_name, io_ctx->vol_size);
+      io_ctx->type = RBDSC_REGISTER_REPLY;
+      m_cache_server->send(sesstion_id, std::string((char*)io_ctx, msg.size()));
+
+      break;
+    }
+    case RBDSC_READ: {
+      // lookup object in local cache store
+      ret = m_object_cache_store->lookup_object(io_ctx->pool_name, io_ctx->vol_name);
+      if (ret < 0) {
+        io_ctx->type = RBDSC_READ_RADOS;
+      } else {
+        io_ctx->type = RBDSC_READ_REPLY;
+      }
+      m_cache_server->send(sesstion_id, std::string((char*)io_ctx, msg.size()));
+
+      break;
+    }
+    
+  }
+}
diff --git a/src/tools/rbd_cache/CacheController.hpp b/src/tools/rbd_cache/CacheController.hpp
new file mode 100644
index 0000000..97113e4
--- /dev/null
+++ b/src/tools/rbd_cache/CacheController.hpp
@@ -0,0 +1,49 @@
+#ifndef CACHE_CONTROLLER_H
+#define CACHE_CONTROLLER_H
+
+#include <thread>
+
+#include "common/Formatter.h"
+#include "common/admin_socket.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/ceph_context.h"
+#include "common/Mutex.h"
+#include "common/WorkQueue.h"
+#include "include/rados/librados.hpp"
+#include "include/rbd/librbd.h"
+#include "include/assert.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/ImageState.h"
+
+#include "CacheControllerSocket.hpp"
+#include "ObjectCacheStore.h"
+
+
+using boost::asio::local::stream_protocol;
+
+class CacheController {
+ public:
+  CacheController(CephContext *cct, const std::vector<const char*> &args);
+  ~CacheController();
+
+  int init();
+
+  int shutdown();
+
+  void handle_signal(int sinnum);
+
+  void run();
+
+  void handle_request(uint64_t sesstion_id, std::string msg);
+
+ private:
+  boost::asio::io_service io_service;
+  CacheServer *m_cache_server;
+  std::vector<const char*> m_args;
+  CephContext *m_cct;
+  ObjectCacheStore *m_object_cache_store;
+  ContextWQ* pcache_op_work_queue;
+};
+
+#endif
diff --git a/src/tools/rbd_cache/CacheControllerSocket.hpp b/src/tools/rbd_cache/CacheControllerSocket.hpp
new file mode 100644
index 0000000..6e1a743
--- /dev/null
+++ b/src/tools/rbd_cache/CacheControllerSocket.hpp
@@ -0,0 +1,125 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CACHE_CONTROLLER_SOCKET_H
+#define CACHE_CONTROLLER_SOCKET_H
+
+#include <cstdio>
+#include <iostream>
+#include <array>
+#include <memory>
+#include <string>
+#include <boost/bind.hpp>
+#include <boost/asio.hpp>
+#include <boost/algorithm/string.hpp>
+#include "CacheControllerSocketCommon.h"
+
+
+using boost::asio::local::stream_protocol;
+
+class session : public std::enable_shared_from_this<session> {
+public:
+  session(uint64_t session_id, boost::asio::io_service& io_service, ProcessMsg processmsg)
+    : session_id(session_id), socket_(io_service), process_msg(processmsg) {}
+
+  stream_protocol::socket& socket() {
+    return socket_;
+  }
+
+  void start() {
+
+    boost::asio::async_read(socket_, boost::asio::buffer(data_),
+                            boost::asio::transfer_exactly(544),
+                            boost::bind(&session::handle_read,
+                            shared_from_this(),
+                            boost::asio::placeholders::error,
+                            boost::asio::placeholders::bytes_transferred));
+
+  }
+
+  void handle_read(const boost::system::error_code& error, size_t bytes_transferred) {
+
+    if (!error) {
+     
+      process_msg(session_id, std::string(data_, bytes_transferred));
+
+    }
+  }
+
+  void handle_write(const boost::system::error_code& error) {
+    if (!error) {
+      socket_.async_read_some(boost::asio::buffer(data_),
+          boost::bind(&session::handle_read,
+            shared_from_this(),
+            boost::asio::placeholders::error,
+            boost::asio::placeholders::bytes_transferred));
+    }
+  }
+
+  void send(std::string msg) {
+
+      boost::asio::async_write(socket_,
+          boost::asio::buffer(msg.c_str(), msg.size()),
+          boost::bind(&session::handle_write,
+            shared_from_this(),
+            boost::asio::placeholders::error));
+
+  }
+
+private:
+  uint64_t session_id;
+  stream_protocol::socket socket_;
+  ProcessMsg process_msg;
+
+  // Buffer used to store data received from the client.
+  //std::array<char, 1024> data_;
+  char data_[1024];
+};
+
+typedef std::shared_ptr<session> session_ptr;
+
+class CacheServer {
+public:
+  CacheServer(boost::asio::io_service& io_service,
+         const std::string& file, ProcessMsg processmsg)
+    : io_service_(io_service),
+      server_process_msg(processmsg),
+      acceptor_(io_service, stream_protocol::endpoint(file))
+  {
+    session_ptr new_session(new session(session_id, io_service_, server_process_msg));
+    acceptor_.async_accept(new_session->socket(),
+        boost::bind(&CacheServer::handle_accept, this, new_session,
+          boost::asio::placeholders::error));
+  }
+
+  void handle_accept(session_ptr new_session,
+      const boost::system::error_code& error)
+  {
+    //TODO(): open librbd snap
+    if (!error) {
+      new_session->start();
+      session_map.emplace(session_id, new_session);
+      session_id++;
+      new_session.reset(new session(session_id, io_service_, server_process_msg));
+      acceptor_.async_accept(new_session->socket(),
+          boost::bind(&CacheServer::handle_accept, this, new_session,
+            boost::asio::placeholders::error));
+    }
+  }
+
+  void send(uint64_t session_id, std::string msg) {
+    auto it = session_map.find(session_id);
+    if (it != session_map.end()) {
+      it->second->send(msg);
+    }
+  }
+
+private:
+  boost::asio::io_service& io_service_;
+  ProcessMsg server_process_msg;
+  stream_protocol::acceptor acceptor_;
+  uint64_t session_id = 1;
+  std::map<uint64_t, session_ptr> session_map;
+};
+
+#endif
diff --git a/src/tools/rbd_cache/CacheControllerSocketClient.hpp b/src/tools/rbd_cache/CacheControllerSocketClient.hpp
new file mode 100644
index 0000000..8e61aa9
--- /dev/null
+++ b/src/tools/rbd_cache/CacheControllerSocketClient.hpp
@@ -0,0 +1,131 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CACHE_CONTROLLER_SOCKET_CLIENT_H
+#define CACHE_CONTROLLER_SOCKET_CLIENT_H
+
+#include <boost/asio.hpp>
+#include <boost/bind.hpp>
+#include <boost/algorithm/string.hpp>
+#include "include/assert.h"
+#include "CacheControllerSocketCommon.h"
+
+
+using boost::asio::local::stream_protocol;
+
+class CacheClient {
+public:
+  CacheClient(boost::asio::io_service& io_service,
+              const std::string& file, ClientProcessMsg processmsg)
+    : io_service_(io_service),
+      io_service_work_(io_service),
+      socket_(io_service),
+      m_client_process_msg(processmsg),
+      ep_(stream_protocol::endpoint(file))
+  {
+     std::thread thd([this](){io_service_.run(); });
+     thd.detach();
+  }
+
+  void run(){
+  } 
+
+  int connect() {
+    try {
+      socket_.connect(ep_);
+    } catch (std::exception& e) {
+      return -1;
+    }
+    connected = true;
+    return 0;
+  }
+
+  int register_volume(std::string pool_name, std::string vol_name, uint64_t vol_size) {
+    // cache controller will init layout
+    rbdsc_req_type_t *message = new rbdsc_req_type_t();
+    message->type = RBDSC_REGISTER;
+    memcpy(message->pool_name, pool_name.c_str(), pool_name.size());
+    memcpy(message->vol_name, vol_name.c_str(), vol_name.size());
+    message->vol_size = vol_size;
+    message->offset = 0;
+    message->length = 0;
+    boost::asio::async_write(socket_,  boost::asio::buffer((char*)message, message->size()),
+        [this](const boost::system::error_code& err, size_t cb) {
+        if (!err) {
+          boost::asio::async_read(socket_, boost::asio::buffer(buffer_),
+              boost::asio::transfer_exactly(544),
+              [this](const boost::system::error_code& err, size_t cb) {
+              if (!err) {
+                m_client_process_msg(std::string(buffer_, cb));
+              } else {
+                  return -1;
+              }
+          });
+        } else {
+          return -1;
+        }
+    });
+
+    return 0;
+  }
+
+  int lookup_object(std::string pool_name, std::string vol_name, std::string object_id, bool* result) {
+    rbdsc_req_type_t *message = new rbdsc_req_type_t();
+    message->type = RBDSC_READ;
+    memcpy(message->pool_name, pool_name.c_str(), pool_name.size());
+    memcpy(message->vol_name, object_id.c_str(), object_id.size());
+    message->vol_size = 0;
+    message->offset = 0;
+    message->length = 0;
+
+    boost::asio::async_write(socket_,  boost::asio::buffer((char*)message, message->size()),
+        [this, result](const boost::system::error_code& err, size_t cb) {
+        if (!err) {
+          get_result(result);
+        } else {
+          return -1;
+        }
+    });
+    std::unique_lock<std::mutex> lk(m);
+    cv.wait(lk);
+    return 0;
+  }
+
+  void get_result(bool* result) {
+    boost::asio::async_read(socket_, boost::asio::buffer(buffer_),
+        boost::asio::transfer_exactly(544),
+        [this, result](const boost::system::error_code& err, size_t cb) {
+        if (!err) {
+            *result = true;
+            cv.notify_one();
+            m_client_process_msg(std::string(buffer_, cb));
+        } else {
+            return -1;
+        }
+    });
+  }
+
+  void handle_connect(const boost::system::error_code& error) {
+    //TODO(): open librbd snap
+  }
+
+  void handle_write(const boost::system::error_code& error) {
+  }
+
+private:
+  boost::asio::io_service& io_service_;
+  boost::asio::io_service::work io_service_work_;
+  stream_protocol::socket socket_;
+  ClientProcessMsg m_client_process_msg;
+  stream_protocol::endpoint ep_;
+  char buffer_[1024];
+  int block_size_ = 1024;
+
+  std::condition_variable cv;
+  std::mutex m;
+
+public:
+  bool connected = false;
+};
+
+#endif
diff --git a/src/tools/rbd_cache/CacheControllerSocketCommon.h b/src/tools/rbd_cache/CacheControllerSocketCommon.h
new file mode 100644
index 0000000..e253bb1
--- /dev/null
+++ b/src/tools/rbd_cache/CacheControllerSocketCommon.h
@@ -0,0 +1,43 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CACHE_CONTROLLER_SOCKET_COMMON_H
+#define CACHE_CONTROLLER_SOCKET_COMMON_H
+
+#define RBDSC_REGISTER         0X11
+#define RBDSC_READ             0X12
+#define RBDSC_LOOKUP           0X13
+#define RBDSC_REGISTER_REPLY   0X14
+#define RBDSC_READ_REPLY       0X15
+#define RBDSC_LOOKUP_REPLY     0X16
+#define RBDSC_READ_RADOS       0X16
+
+typedef std::function<void(uint64_t, std::string)> ProcessMsg;
+typedef std::function<void(std::string)> ClientProcessMsg;
+typedef uint8_t rbdsc_req_type;
+struct rbdsc_req_type_t {
+  rbdsc_req_type type;
+  uint64_t vol_size;
+  uint64_t offset;
+  uint64_t length;
+  char pool_name[256];
+  char vol_name[256];
+
+  uint64_t size() {
+    return sizeof(rbdsc_req_type_t);
+  }
+
+  std::string to_buffer() {
+    std::stringstream ss;
+    ss << type;
+    ss << vol_size;
+    ss << offset;
+    ss << length;
+    ss << pool_name;
+    ss << vol_name;
+
+    return ss.str();
+  }
+};
+
+#endif
diff --git a/src/tools/rbd_cache/ObjectCacheStore.cc b/src/tools/rbd_cache/ObjectCacheStore.cc
new file mode 100644
index 0000000..90b407c
--- /dev/null
+++ b/src/tools/rbd_cache/ObjectCacheStore.cc
@@ -0,0 +1,147 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "ObjectCacheStore.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_cache
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::cache::ObjectCacheStore: " << this << " " \
+                           << __func__ << ": "
+
+
+ObjectCacheStore::ObjectCacheStore(CephContext *cct, ContextWQ* work_queue)
+      : m_cct(cct), m_work_queue(work_queue),
+        m_cache_table_lock("rbd::cache::ObjectCacheStore"),
+        m_rados(new librados::Rados()) {
+}
+
+ObjectCacheStore::~ObjectCacheStore() {
+
+}
+
+int ObjectCacheStore::init(bool reset) {
+
+  int ret = m_rados->init_with_context(m_cct);
+  if(ret < 0) {
+    lderr(m_cct) << "fail to init Ceph context" << dendl;
+    return ret;
+  }
+
+  ret = m_rados->connect();
+  if(ret < 0 ) {
+    lderr(m_cct) << "fail to conect to cluster" << dendl;
+    return ret;
+  }
+  //TODO(): check existing cache objects
+  return ret;
+}
+
+int ObjectCacheStore::do_promote(std::string pool_name, std::string object_name) {
+  int ret = 0;
+  std::string cache_file_name =  pool_name + object_name;
+
+  if (m_ioctxs.find(pool_name) == m_ioctxs.end()) {
+    librados::IoCtx* io_ctx = new librados::IoCtx();
+    ret = m_rados->ioctx_create(pool_name.c_str(), *io_ctx);
+    if (ret < 0) {
+      lderr(m_cct) << "fail to create ioctx" << dendl;
+      assert(0);
+    }
+    m_ioctxs.emplace(pool_name, io_ctx); 
+  }
+
+  assert(m_ioctxs.find(pool_name) != m_ioctxs.end());
+  
+  librados::IoCtx* ioctx = m_ioctxs[pool_name]; 
+
+  //promoting: update metadata 
+  {
+    Mutex::Locker locker(m_cache_table_lock);
+    m_cache_table.emplace(cache_file_name, PROMOTING);
+  }
+
+  librados::bufferlist read_buf;      
+  int object_size = 4096*1024; //TODO(): read config from image metadata
+
+  //TODO(): async promote
+  ret = promote_object(ioctx, object_name, read_buf, object_size);
+  if (ret == -ENOENT) {
+    read_buf.append(std::string(object_size, '0'));
+    ret = 0;
+  }
+
+  if( ret < 0) {
+    lderr(m_cct) << "fail to read from rados" << dendl;
+    return ret;
+  }
+
+  // persistent to cache
+  os::CacheStore::SyncFile cache_file(m_cct, cache_file_name);
+  cache_file.open();
+  ret = cache_file.write_object_to_file(read_buf, object_size);
+  
+  assert(m_cache_table.find(cache_file_name) != m_cache_table.end()); 
+
+  // update metadata
+  {
+    Mutex::Locker locker(m_cache_table_lock);
+    m_cache_table.emplace(cache_file_name, PROMOTED);
+  }
+
+  return ret;
+
+}
+ 
+int ObjectCacheStore::lookup_object(std::string pool_name, std::string object_name) {
+
+  std::string cache_file_name =  pool_name + object_name;
+  {
+    Mutex::Locker locker(m_cache_table_lock);
+
+    auto it = m_cache_table.find(cache_file_name);
+    if (it != m_cache_table.end()) {
+
+      if (it->second == PROMOTING) {
+        return -1;
+      } else if (it->second == PROMOTED) {
+        return 0;
+      } else {
+        assert(0);
+      }
+    }
+  }
+
+  int ret = do_promote(pool_name, object_name);
+
+  return ret;
+}
+
+int ObjectCacheStore::shutdown() {
+  m_rados->shutdown();
+  return 0;
+}
+
+int ObjectCacheStore::init_cache(std::string vol_name, uint64_t vol_size) {
+  return 0;
+}
+
+int ObjectCacheStore::lock_cache(std::string vol_name) {
+  return 0;
+}
+
+int ObjectCacheStore::promote_object(librados::IoCtx* ioctx, std::string object_name, librados::bufferlist read_buf, uint64_t read_len) {
+  int ret;
+
+  librados::AioCompletion* read_completion = librados::Rados::aio_create_completion(); 
+
+  ret = ioctx->aio_read(object_name, read_completion, &read_buf, read_len, 0);
+  if(ret < 0) {
+    lderr(m_cct) << "fail to read from rados" << dendl;
+    return ret;
+  }
+  read_completion->wait_for_complete();
+  ret = read_completion->get_return_value();
+  return ret;
+  
+}
diff --git a/src/tools/rbd_cache/ObjectCacheStore.h b/src/tools/rbd_cache/ObjectCacheStore.h
new file mode 100644
index 0000000..12f8399
--- /dev/null
+++ b/src/tools/rbd_cache/ObjectCacheStore.h
@@ -0,0 +1,65 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef OBJECT_CACHE_STORE_H
+#define OBJECT_CACHE_STORE_H
+
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/ceph_context.h"
+#include "common/Mutex.h"
+#include "include/rados/librados.hpp"
+#include "include/rbd/librbd.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/ImageState.h"
+#include "os/CacheStore/SyncFile.h"
+
+using librados::Rados;
+using librados::IoCtx;
+
+typedef shared_ptr<librados::Rados> RadosRef;
+typedef shared_ptr<librados::IoCtx> IoCtxRef;
+
+class ObjectCacheStore 
+{
+  public:
+    ObjectCacheStore(CephContext *cct, ContextWQ* work_queue);
+    ~ObjectCacheStore();
+
+    int init(bool reset);
+
+    int shutdown();
+
+    int lookup_object(std::string pool_name, std::string object_name);
+
+    int init_cache(std::string vol_name, uint64_t vol_size);
+
+    int lock_cache(std::string vol_name);
+
+  private:
+    int _evict_object();
+
+    int do_promote(std::string pool_name, std::string object_name);
+
+    int promote_object(librados::IoCtx*, std::string object_name,
+                       librados::bufferlist read_buf,
+                       uint64_t length);
+
+    enum {
+      PROMOTING = 0, 
+      PROMOTED, 
+    };
+
+    CephContext *m_cct;
+    ContextWQ* m_work_queue;
+    Mutex m_cache_table_lock;
+    RadosRef m_rados;
+
+    std::map<std::string, uint8_t> m_cache_table;
+
+    std::map<std::string, librados::IoCtx*> m_ioctxs;
+
+    os::CacheStore::SyncFile *m_cache_file;
+};
+
+#endif
diff --git a/src/tools/rbd_cache/main.cc b/src/tools/rbd_cache/main.cc
new file mode 100644
index 0000000..336a581
--- /dev/null
+++ b/src/tools/rbd_cache/main.cc
@@ -0,0 +1,85 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "common/ceph_argparse.h"
+#include "common/config.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "global/global_init.h"
+#include "global/signal_handler.h"
+#include "CacheController.hpp"
+
+#include <vector>
+
+CacheController *cachectl = nullptr;
+
+void usage() {
+  std::cout << "usage: cache controller [options...]" << std::endl;
+  std::cout << "options:\n";
+  std::cout << "  -m monaddress[:port]      connect to specified monitor\n";
+  std::cout << "  --keyring=<path>          path to keyring for local cluster\n";
+  std::cout << "  --log-file=<logfile>       file to log debug output\n";
+  std::cout << "  --debug-rbd-cachecontroller=<log-level>/<memory-level>  set rbd-mirror debug level\n";
+  generic_server_usage();
+}
+
+static void handle_signal(int signum)
+{
+  if (cachectl)
+    cachectl->handle_signal(signum);
+}
+
+int main(int argc, const char **argv)
+{
+  std::vector<const char*> args;
+  env_to_vec(args);
+  argv_to_vec(argc, argv, args);
+
+  auto cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT,
+			 CODE_ENVIRONMENT_DAEMON,
+			 CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS);
+
+  for (auto i = args.begin(); i != args.end(); ++i) {
+    if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
+      usage();
+      return EXIT_SUCCESS;
+    }
+  }
+
+  if (g_conf->daemonize) {
+    global_init_daemonize(g_ceph_context);
+  }
+  g_ceph_context->enable_perf_counter();
+
+  common_init_finish(g_ceph_context);
+
+  init_async_signal_handler();
+  register_async_signal_handler(SIGHUP, sighup_handler);
+  register_async_signal_handler_oneshot(SIGINT, handle_signal);
+  register_async_signal_handler_oneshot(SIGTERM, handle_signal);
+
+  std::vector<const char*> cmd_args;
+  argv_to_vec(argc, argv, cmd_args);
+
+  // disable unnecessary librbd cache
+  g_ceph_context->_conf->set_val_or_die("rbd_cache", "false");
+
+  cachectl = new CacheController(g_ceph_context, cmd_args);
+  int r = cachectl->init();
+  if (r < 0) {
+    std::cerr << "failed to initialize: " << cpp_strerror(r) << std::endl;
+    goto cleanup;
+  }
+
+  cachectl->run();
+
+ cleanup:
+  unregister_async_signal_handler(SIGHUP, sighup_handler);
+  unregister_async_signal_handler(SIGINT, handle_signal);
+  unregister_async_signal_handler(SIGTERM, handle_signal);
+  shutdown_async_signal_handler();
+
+  delete cachectl;
+
+  return r < 0 ? EXIT_SUCCESS : EXIT_FAILURE;
+}
-- 
2.7.4