Handoff checksum Implementation (#7523)

Summary:
in PR https://github.com/facebook/rocksdb/issues/7419 , we introduce the new Append and PositionedAppend APIs to WritableFile at File System, which enable RocksDB to pass the data verification information (e.g., checksum of the data) to the lower layer. In this PR, we use the new API in WritableFileWriter, such that the file created via WritableFileWrite can pass the checksum to the storage layer. To control which types file should apply the checksum handoff, we add checksum_handoff_file_types to DBOptions. User can use this option to control which file types (Currently supported file tyes: kLogFile, kTableFile, kDescriptorFile.) should use the new Append and PositionedAppend APIs to handoff the verification information.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7523

Test Plan: add new unit test, pass make check/ make asan_check

Reviewed By: pdillinger

Differential Revision: D24313271

Pulled By: zhichao-cao

fbshipit-source-id: aafd69091ae85c3318e3e17cbb96fe7338da11d0

6 files changed, 12 insertions, 2 deletions

diff --git a/options/cf_options.cc b/options/cf_options.cc
index c436dd312..7e680fd88 100644
--- a/options/cf_options.cc
+++ b/options/cf_options.cc
@@ -846,7 +846,8 @@ ImmutableCFOptions::ImmutableCFOptions(const ImmutableDBOptions& db_options,
       file_checksum_gen_factory(db_options.file_checksum_gen_factory.get()),
       sst_partitioner_factory(cf_options.sst_partitioner_factory),
       allow_data_in_errors(db_options.allow_data_in_errors),
-      db_host_id(db_options.db_host_id) {}
+      db_host_id(db_options.db_host_id),
+      checksum_handoff_file_types(db_options.checksum_handoff_file_types) {}
 
 // Multiple two operands. If they overflow, return op1.
 uint64_t MultiplyCheckOverflow(uint64_t op1, double op2) {
diff --git a/options/cf_options.h b/options/cf_options.h
index c9e8f068f..7bbea71e4 100644
--- a/options/cf_options.h
+++ b/options/cf_options.h
@@ -126,6 +126,8 @@ struct ImmutableCFOptions {
   bool allow_data_in_errors;
 
   std::string db_host_id;
+
+  FileTypeSet checksum_handoff_file_types;
 };
 
 struct MutableCFOptions {
diff --git a/options/db_options.cc b/options/db_options.cc
index 3733d448c..8e587117b 100644
--- a/options/db_options.cc
+++ b/options/db_options.cc
@@ -136,6 +136,7 @@ static std::unordered_map<std::string, OptionTypeInfo>
           std::shared_ptr<Statistics> statistics;
           std::vector<DbPath> db_paths;
           std::vector<std::shared_ptr<EventListener>> listeners;
+          FileTypeSet checksum_handoff_file_types;
          */
         {"advise_random_on_open",
          {offsetof(struct ImmutableDBOptions, advise_random_on_open),
@@ -580,7 +581,8 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
       max_bgerror_resume_count(options.max_bgerror_resume_count),
       bgerror_resume_retry_interval(options.bgerror_resume_retry_interval),
       allow_data_in_errors(options.allow_data_in_errors),
-      db_host_id(options.db_host_id) {
+      db_host_id(options.db_host_id),
+      checksum_handoff_file_types(options.checksum_handoff_file_types) {
 }
 
 void ImmutableDBOptions::Dump(Logger* log) const {
diff --git a/options/db_options.h b/options/db_options.h
index 42a58e256..a454e42b0 100644
--- a/options/db_options.h
+++ b/options/db_options.h
@@ -93,6 +93,7 @@ struct ImmutableDBOptions {
   uint64_t bgerror_resume_retry_interval;
   bool allow_data_in_errors;
   std::string db_host_id;
+  FileTypeSet checksum_handoff_file_types;
 };
 
 struct MutableDBOptions {
diff --git a/options/options_helper.cc b/options/options_helper.cc
index 02139a62b..e4e8d4ee8 100644
--- a/options/options_helper.cc
+++ b/options/options_helper.cc
@@ -168,6 +168,8 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
       immutable_db_options.bgerror_resume_retry_interval;
   options.db_host_id = immutable_db_options.db_host_id;
   options.allow_data_in_errors = immutable_db_options.allow_data_in_errors;
+  options.checksum_handoff_file_types =
+      immutable_db_options.checksum_handoff_file_types;
   return options;
 }
 
diff --git a/options/options_settable_test.cc b/options/options_settable_test.cc
index 5e0d402fd..a1fc17514 100644
--- a/options/options_settable_test.cc
+++ b/options/options_settable_test.cc
@@ -227,6 +227,8 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
       {offsetof(struct DBOptions, file_checksum_gen_factory),
        sizeof(std::shared_ptr<FileChecksumGenFactory>)},
       {offsetof(struct DBOptions, db_host_id), sizeof(std::string)},
+      {offsetof(struct DBOptions, checksum_handoff_file_types),
+       sizeof(FileTypeSet)},
   };
 
   char* options_ptr = new char[sizeof(DBOptions)];