Attempt to recover from db with missing table files (#6334)

Summary:
There are situations when RocksDB tries to recover, but the db is in an inconsistent state due to SST files referenced in the MANIFEST being missing. In this case, previous RocksDB will just fail the recovery and return a non-ok status.
This PR enables another possibility. During recovery, RocksDB checks possible MANIFEST files, and try to recover to the most recent state without missing table file. `VersionSet::Recover()` applies version edits incrementally and "materializes" a version only when this version does not reference any missing table file. After processing the entire MANIFEST, the version created last will be the latest version.
`DBImpl::Recover()` calls `VersionSet::Recover()`. Afterwards, WAL replay will *not* be performed.
To use this capability, set `options.best_efforts_recovery = true` when opening the db. Best-efforts recovery is currently incompatible with atomic flush.

Test plan (on devserver):
```
$make check
$COMPILE_WITH_ASAN=1 make all && make check
```
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6334

Reviewed By: anand1976

Differential Revision: D19778960

Pulled By: riversand963

fbshipit-source-id: c27ea80f29bc952e7d3311ecf5ee9c54393b40a8

4 files changed, 11 insertions, 2 deletions

diff --git a/options/db_options.cc b/options/db_options.cc
index 0dcc8ef79..478897786 100644
--- a/options/db_options.cc
+++ b/options/db_options.cc
@@ -95,7 +95,8 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
       persist_stats_to_disk(options.persist_stats_to_disk),
       write_dbid_to_manifest(options.write_dbid_to_manifest),
       log_readahead_size(options.log_readahead_size),
-      sst_file_checksum_func(options.sst_file_checksum_func) {
+      sst_file_checksum_func(options.sst_file_checksum_func),
+      best_efforts_recovery(options.best_efforts_recovery) {
 }
 
 void ImmutableDBOptions::Dump(Logger* log) const {
@@ -250,6 +251,8 @@ void ImmutableDBOptions::Dump(Logger* log) const {
                    sst_file_checksum_func
                        ? sst_file_checksum_func->Name()
                        : kUnknownFileChecksumFuncName.c_str());
+  ROCKS_LOG_HEADER(log, "                Options.best_efforts_recovery: %d",
+                   static_cast<int>(best_efforts_recovery));
 }
 
 MutableDBOptions::MutableDBOptions()
diff --git a/options/db_options.h b/options/db_options.h
index cba298a21..3ab6b73eb 100644
--- a/options/db_options.h
+++ b/options/db_options.h
@@ -88,6 +88,7 @@ struct ImmutableDBOptions {
   bool write_dbid_to_manifest;
   size_t log_readahead_size;
   std::shared_ptr<FileChecksumFunc> sst_file_checksum_func;
+  bool best_efforts_recovery;
 };
 
 struct MutableDBOptions {
diff --git a/options/options_helper.cc b/options/options_helper.cc
index bc36dbcfe..3ebbcee4e 100644
--- a/options/options_helper.cc
+++ b/options/options_helper.cc
@@ -145,6 +145,7 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
       immutable_db_options.avoid_unnecessary_blocking_io;
   options.log_readahead_size = immutable_db_options.log_readahead_size;
   options.sst_file_checksum_func = immutable_db_options.sst_file_checksum_func;
+  options.best_efforts_recovery = immutable_db_options.best_efforts_recovery;
   return options;
 }
 
@@ -1681,6 +1682,9 @@ std::unordered_map<std::string, OptionTypeInfo>
         {"log_readahead_size",
          {offsetof(struct DBOptions, log_readahead_size), OptionType::kSizeT,
           OptionVerificationType::kNormal, false, 0}},
+        {"best_efforts_recovery",
+         {offsetof(struct DBOptions, best_efforts_recovery),
+          OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
 };
 
 std::unordered_map<std::string, BlockBasedTableOptions::IndexType>
diff --git a/options/options_settable_test.cc b/options/options_settable_test.cc
index c5fee7b3e..057d772bf 100644
--- a/options/options_settable_test.cc
+++ b/options/options_settable_test.cc
@@ -303,7 +303,8 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
                              "atomic_flush=false;"
                              "avoid_unnecessary_blocking_io=false;"
                              "log_readahead_size=0;"
-                             "write_dbid_to_manifest=false",
+                             "write_dbid_to_manifest=false;"
+                             "best_efforts_recovery=false",
                              new_options));
 
   ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions),