Add APIs to query secondary cache capacity and usage for TieredCache (#12011)

Summary:
In `TieredCache`, the underlying compressed secondary cache is hidden from the user. So we need a way to query the capacity, as well as the portion of cache reservation charged to the compressed secondary cache.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/12011

Test Plan: Update the unit tests

Reviewed By: akankshamahajan15

Differential Revision: D50651943

Pulled By: anand1976

fbshipit-source-id: 06d1cb5edb75a790c919bce718e2ff65f5908220

7 files changed, 56 insertions, 1 deletions

diff --git a/cache/compressed_secondary_cache_test.cc b/cache/compressed_secondary_cache_test.cc
index 71702b29f..84633941d 100644
--- a/cache/compressed_secondary_cache_test.cc
+++ b/cache/compressed_secondary_cache_test.cc
@@ -1276,6 +1276,13 @@ TEST_P(CompressedSecCacheTestWithTiered, DynamicUpdateWithReservation) {
   ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
   ASSERT_EQ(sec_capacity, (30 << 20));
 
+  ASSERT_OK(tiered_cache->GetSecondaryCacheCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, 30 << 20);
+  size_t sec_usage;
+  ASSERT_OK(tiered_cache->GetSecondaryCachePinnedUsage(sec_usage));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_usage, 3 << 20,
+               GetPercent(3 << 20, 1));
+
   ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.39));
   EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (45 << 20),
                GetPercent(45 << 20, 1));
diff --git a/cache/secondary_cache_adapter.cc b/cache/secondary_cache_adapter.cc
index 70571f5b2..84b4437e8 100644
--- a/cache/secondary_cache_adapter.cc
+++ b/cache/secondary_cache_adapter.cc
@@ -489,6 +489,29 @@ void CacheWithSecondaryAdapter::SetCapacity(size_t capacity) {
   }
 }
 
+Status CacheWithSecondaryAdapter::GetSecondaryCacheCapacity(
+    size_t& size) const {
+  return secondary_cache_->GetCapacity(size);
+}
+
+Status CacheWithSecondaryAdapter::GetSecondaryCachePinnedUsage(
+    size_t& size) const {
+  Status s;
+  if (distribute_cache_res_) {
+    MutexLock m(&mutex_);
+    size_t capacity = 0;
+    s = secondary_cache_->GetCapacity(capacity);
+    if (s.ok()) {
+      size = capacity - pri_cache_res_->GetTotalMemoryUsed();
+    } else {
+      size = 0;
+    }
+  } else {
+    size = 0;
+  }
+  return s;
+}
+
 // Update the secondary/primary allocation ratio (remember, the primary
 // capacity is the total memory budget when distribute_cache_res_ is true).
 // When the ratio changes, we may accumulate some error in the calculations
diff --git a/cache/secondary_cache_adapter.h b/cache/secondary_cache_adapter.h
index 0d5f2d6ea..6b06d0829 100644
--- a/cache/secondary_cache_adapter.h
+++ b/cache/secondary_cache_adapter.h
@@ -47,6 +47,10 @@ class CacheWithSecondaryAdapter : public CacheWrapper {
 
   void SetCapacity(size_t capacity) override;
 
+  Status GetSecondaryCacheCapacity(size_t& size) const override;
+
+  Status GetSecondaryCachePinnedUsage(size_t& size) const override;
+
   Status UpdateCacheReservationRatio(double ratio);
 
   Status UpdateAdmissionPolicy(TieredAdmissionPolicy adm_policy);
@@ -81,7 +85,7 @@ class CacheWithSecondaryAdapter : public CacheWrapper {
   // Fraction of a cache memory reservation to be assigned to the secondary
   // cache
   std::atomic<double> sec_cache_res_ratio_;
-  port::Mutex mutex_;
+  mutable port::Mutex mutex_;
 #ifndef NDEBUG
   bool ratio_changed_ = false;
 #endif
diff --git a/cache/sharded_cache.cc b/cache/sharded_cache.cc
index 322b59226..b270df751 100644
--- a/cache/sharded_cache.cc
+++ b/cache/sharded_cache.cc
@@ -83,6 +83,16 @@ size_t ShardedCacheBase::GetCapacity() const {
   return capacity_;
 }
 
+Status ShardedCacheBase::GetSecondaryCacheCapacity(size_t& size) const {
+  size = 0;
+  return Status::OK();
+}
+
+Status ShardedCacheBase::GetSecondaryCachePinnedUsage(size_t& size) const {
+  size = 0;
+  return Status::OK();
+}
+
 bool ShardedCacheBase::HasStrictCapacityLimit() const {
   MutexLock l(&config_mutex_);
   return strict_capacity_limit_;
diff --git a/cache/sharded_cache.h b/cache/sharded_cache.h
index 39042137f..b7ef723a1 100644
--- a/cache/sharded_cache.h
+++ b/cache/sharded_cache.h
@@ -99,6 +99,8 @@ class ShardedCacheBase : public Cache {
 
   bool HasStrictCapacityLimit() const override;
   size_t GetCapacity() const override;
+  Status GetSecondaryCacheCapacity(size_t& size) const override;
+  Status GetSecondaryCachePinnedUsage(size_t& size) const override;
 
   using Cache::GetUsage;
   size_t GetUsage(Handle* handle) const override;
diff --git a/include/rocksdb/advanced_cache.h b/include/rocksdb/advanced_cache.h
index b5dcc3d49..77f1f5ce1 100644
--- a/include/rocksdb/advanced_cache.h
+++ b/include/rocksdb/advanced_cache.h
@@ -375,6 +375,14 @@ class Cache {
   // Returns the helper for the specified entry.
   virtual const CacheItemHelper* GetCacheItemHelper(Handle* handle) const = 0;
 
+  virtual Status GetSecondaryCacheCapacity(size_t& /*size*/) const {
+    return Status::NotSupported();
+  }
+
+  virtual Status GetSecondaryCachePinnedUsage(size_t& /*size*/) const {
+    return Status::NotSupported();
+  }
+
   // Call this on shutdown if you want to speed it up. Cache will disown
   // any underlying data and will not free it on delete. This call will leak
   // memory - call this only if you're shutting down the process.
diff --git a/unreleased_history/public_api_changes/tiered_cache_capacity_and_usage.md b/unreleased_history/public_api_changes/tiered_cache_capacity_and_usage.md
new file mode 100644
index 000000000..e281d069d
--- /dev/null
+++ b/unreleased_history/public_api_changes/tiered_cache_capacity_and_usage.md
@@ -0,0 +1 @@
+Add new Cache APIs GetSecondaryCacheCapacity() and GetSecondaryCachePinnedUsage() to return the configured capacity, and cache reservation charged to the secondary cache.