From b7b81562c76011abe05930330915a5ba423964e4 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Thu, 19 Apr 2018 22:54:36 +0800 Subject: [PATCH 01/10] librbd: shared persistent read-only rbd cache This patch introduces introduces RBD shared persistent RO cache which can provide client-side sharing cache for rbd clone/snapshot case. The key componenets are: - RBD cache daemon runs on each compute node to control the shared cache state - Read-only blocks from parent image(s) are cached in a shared area on compute node(s) - Object level dispatcher inside librbd that can do RPC with cache daemon to lookup the cache - Reads are served from the shared cache until the first COW request - Policy to control promotion/evication of the shared cache The general IO flow is: 0) Parent image would register themselfs when initializing 1) When read request on cloned image flows to parent image, it will check with the cache daemon if the rarget object is ready 2) Cache daemon receives the lookup request: a) if the target object is promoted, daemon will ack with "read_from_cache" b) if it is not promoted, daemon will check the policy whether to promote: - if yes, daemon will do the promiton then ack with "read_from_cache" - if no, daemon will ack with "read_from_rados" 3) the read reqeust contines to do read from cache/rados based on the ack Signed-off-by: Yuan Zhou --- src/common/options.cc | 8 ++ src/librbd/CMakeLists.txt | 4 +- src/librbd/ImageCtx.cc | 5 +- src/librbd/ImageCtx.h | 3 + src/librbd/cache/SharedPersistentObjectCacher.cc | 61 ++++++++ src/librbd/cache/SharedPersistentObjectCacher.h | 45 ++++++ .../SharedPersistentObjectCacherObjectDispatch.cc | 154 +++++++++++++++++++++ .../SharedPersistentObjectCacherObjectDispatch.h | 127 +++++++++++++++++ src/librbd/image/OpenRequest.cc | 12 +- src/librbd/io/Types.h | 1 + src/os/CacheStore/SyncFile.cc | 110 +++++++++++++++ src/os/CacheStore/SyncFile.h | 74 ++++++++++ src/test/librbd/test_mirroring.cc | 1 + src/test/rbd_mirror/test_ImageReplayer.cc | 2 + src/test/rbd_mirror/test_fixture.cc | 1 + src/tools/CMakeLists.txt | 1 + src/tools/rbd_cache/CMakeLists.txt | 9 ++ src/tools/rbd_cache/CacheController.cc | 105 ++++++++++++++ src/tools/rbd_cache/CacheController.hpp | 49 +++++++ src/tools/rbd_cache/CacheControllerSocket.hpp | 125 +++++++++++++++++ .../rbd_cache/CacheControllerSocketClient.hpp | 131 ++++++++++++++++++ src/tools/rbd_cache/CacheControllerSocketCommon.h | 43 ++++++ src/tools/rbd_cache/ObjectCacheStore.cc | 147 ++++++++++++++++++++ src/tools/rbd_cache/ObjectCacheStore.h | 65 +++++++++ src/tools/rbd_cache/main.cc | 85 ++++++++++++ 25 files changed, 1365 insertions(+), 3 deletions(-) create mode 100644 src/librbd/cache/SharedPersistentObjectCacher.cc create mode 100644 src/librbd/cache/SharedPersistentObjectCacher.h create mode 100644 src/librbd/cache/SharedPersistentObjectCacherObjectDispatch.cc create mode 100644 src/librbd/cache/SharedPersistentObjectCacherObjectDispatch.h create mode 100644 src/os/CacheStore/SyncFile.cc create mode 100644 src/os/CacheStore/SyncFile.h create mode 100644 src/tools/rbd_cache/CMakeLists.txt create mode 100644 src/tools/rbd_cache/CacheController.cc create mode 100644 src/tools/rbd_cache/CacheController.hpp create mode 100644 src/tools/rbd_cache/CacheControllerSocket.hpp create mode 100644 src/tools/rbd_cache/CacheControllerSocketClient.hpp create mode 100644 src/tools/rbd_cache/CacheControllerSocketCommon.h create mode 100644 src/tools/rbd_cache/ObjectCacheStore.cc create mode 100644 src/tools/rbd_cache/ObjectCacheStore.h create mode 100644 src/tools/rbd_cache/main.cc diff --git a/src/common/options.cc b/src/common/options.cc index c5afe4c..7839a31 100644 --- a/src/common/options.cc +++ b/src/common/options.cc @@ -6357,6 +6357,14 @@ static std::vector