diff options
author | Evan Jones <evan.jones@datadoghq.com> | 2024-06-07 16:53:43 -0700 |
---|---|---|
committer | Facebook GitHub Bot <facebook-github-bot@users.noreply.github.com> | 2024-06-07 16:53:43 -0700 |
commit | 32e6825bc6a18d9e61de375b62139cc4a0f0fe36 (patch) | |
tree | 0d3bbe20984aca72dc177038753c0cf5b7db4a80 | |
parent | b34cef57b798520791312f2f40681c4d12d5d33c (diff) |
c.h: Add GetDbIdentity, Options::write_dbid_to_manifest (#12736)
Summary:
The write_dbid_to_manifest option is documented as "We recommend setting this flag to true". However, there is no way to set this flag from the C API.
Add the following functions to the C API:
* rocksdb_get_db_identity
* rocksdb_options_get_write_dbid_to_manifest
* rocksdb_options_set_write_dbid_to_manifest
Add a test that this option preserves the ID across checkpoints.
c.cc:
* Remove outdated comments about missing C API functions that exist.
* Document that CopyString is intended for binary data and is not NUL terminated.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/12736
Reviewed By: ltamasi
Differential Revision: D58202117
Pulled By: ajkr
fbshipit-source-id: 707b110df5c4bd118d65548327428a53a9dc3019
-rw-r--r-- | db/c.cc | 28 | ||||
-rw-r--r-- | db/c_test.c | 96 | ||||
-rw-r--r-- | include/rocksdb/c.h | 12 |
3 files changed, 130 insertions, 6 deletions
@@ -547,6 +547,7 @@ static bool SaveError(char** errptr, const Status& s) { return true; } +// Copies str to a new malloc()-ed buffer. The buffer is not NUL terminated. static char* CopyString(const std::string& str) { char* result = reinterpret_cast<char*>(malloc(sizeof(char) * str.size())); memcpy(result, str.data(), sizeof(char) * str.size()); @@ -1392,6 +1393,18 @@ char* rocksdb_get_cf_with_ts(rocksdb_t* db, return result; } +char* rocksdb_get_db_identity(rocksdb_t* db, size_t* id_len) { + std::string identity_tmp; + Status s = db->rep->GetDbIdentity(identity_tmp); + if (!s.ok()) { + *id_len = 0; + return nullptr; + } + + *id_len = identity_tmp.size(); + return CopyString(identity_tmp); +} + void rocksdb_multi_get(rocksdb_t* db, const rocksdb_readoptions_t* options, size_t num_keys, const char* const* keys_list, const size_t* keys_list_sizes, char** values_list, @@ -3946,6 +3959,16 @@ void rocksdb_options_set_plain_table_factory( opt->rep.table_factory.reset(factory); } +unsigned char rocksdb_options_get_write_dbid_to_manifest( + rocksdb_options_t* opt) { + return opt->rep.write_dbid_to_manifest; +} + +void rocksdb_options_set_write_dbid_to_manifest( + rocksdb_options_t* opt, unsigned char write_dbid_to_manifest) { + opt->rep.write_dbid_to_manifest = write_dbid_to_manifest; +} + void rocksdb_options_set_max_successive_merges(rocksdb_options_t* opt, size_t v) { opt->rep.max_successive_merges = v; @@ -4325,13 +4348,8 @@ void rocksdb_perfcontext_destroy(rocksdb_perfcontext_t* context) { /* TODO: -DB::OpenForReadOnly -DB::KeyMayExist DB::GetOptions DB::GetSortedWalFiles -DB::GetLatestSequenceNumber -DB::GetUpdatesSince -DB::GetDbIdentity DB::RunManualCompaction custom cache table_properties_collectors diff --git a/db/c_test.c b/db/c_test.c index 81291dce5..fd367aeb7 100644 --- a/db/c_test.c +++ b/db/c_test.c @@ -798,6 +798,11 @@ int main(int argc, char** argv) { rocksdb_delete(db, woptions, "foo", 3, &err); CheckNoError(err); + // get the identity before the backup + size_t before_db_id_len = 0; + char* before_db_id = rocksdb_get_db_identity(db, &before_db_id_len); + CheckCondition(before_db_id_len == 36); + rocksdb_close(db); rocksdb_destroy_db(options, dbname, &err); @@ -818,6 +823,14 @@ int main(int argc, char** argv) { CheckGet(db, roptions, "foo", "hello"); + // the db_identity after the backup is different + size_t after_db_id_len = 0; + char* after_db_id = rocksdb_get_db_identity(db, &after_db_id_len); + CheckCondition(after_db_id_len == 36); + CheckCondition(memcmp(after_db_id, before_db_id, after_db_id_len) != 0); + Free(&before_db_id); + Free(&after_db_id); + rocksdb_backup_engine_close(be); } @@ -833,6 +846,14 @@ int main(int argc, char** argv) { rocksdb_checkpoint_create(checkpoint, dbcheckpointname, 0, &err); CheckNoError(err); + rocksdb_checkpoint_object_destroy(checkpoint); + checkpoint = NULL; + + // get the identity before the checkpoint + size_t before_db_id_len = 0; + char* before_db_id = rocksdb_get_db_identity(db, &before_db_id_len); + CheckCondition(before_db_id_len == 36); + // start a new database from the checkpoint rocksdb_close(db); rocksdb_options_set_error_if_exists(options, 0); @@ -841,7 +862,13 @@ int main(int argc, char** argv) { CheckGet(db, roptions, "foo", "hello"); - rocksdb_checkpoint_object_destroy(checkpoint); + // the db_identity after the checkpoint is different + size_t after_db_id_len = 0; + char* after_db_id = rocksdb_get_db_identity(db, &after_db_id_len); + CheckCondition(after_db_id_len == 36); + CheckCondition(memcmp(after_db_id, before_db_id, after_db_id_len) != 0); + Free(&before_db_id); + Free(&after_db_id); rocksdb_close(db); rocksdb_destroy_db(options, dbcheckpointname, &err); @@ -852,6 +879,73 @@ int main(int argc, char** argv) { rocksdb_options_set_error_if_exists(options, 1); } + StartPhase("checkpoint_db_id_in_manifest"); + { + // create new DB with set_write_dbid_to_manifest=true + // db_identity is now the same across checkpoints + rocksdb_close(db); + db = NULL; + + rocksdb_options_t* options_dbid_in_manifest = rocksdb_options_create(); + rocksdb_options_set_create_if_missing(options_dbid_in_manifest, 1); + + unsigned char write_to_manifest = + rocksdb_options_get_write_dbid_to_manifest(options_dbid_in_manifest); + CheckCondition(!write_to_manifest); + rocksdb_options_set_write_dbid_to_manifest(options_dbid_in_manifest, true); + CheckCondition(!write_to_manifest); + write_to_manifest = + rocksdb_options_get_write_dbid_to_manifest(options_dbid_in_manifest); + CheckCondition(write_to_manifest); + + db = rocksdb_open(options_dbid_in_manifest, dbbackupname, &err); + CheckNoError(err); + + rocksdb_checkpoint_t* checkpoint = + rocksdb_checkpoint_object_create(db, &err); + CheckNoError(err); + + rocksdb_checkpoint_create(checkpoint, dbcheckpointname, 0, &err); + CheckNoError(err); + + rocksdb_checkpoint_object_destroy(checkpoint); + checkpoint = NULL; + + // get the identity before the backup + size_t before_db_id_len = 0; + char* before_db_id = rocksdb_get_db_identity(db, &before_db_id_len); + CheckCondition(before_db_id_len == 36); + + // open the checkpoint + rocksdb_close(db); + rocksdb_destroy_db(options_dbid_in_manifest, dbbackupname, &err); + CheckNoError(err); + + rocksdb_options_set_error_if_exists(options_dbid_in_manifest, 0); + db = rocksdb_open(options_dbid_in_manifest, dbcheckpointname, &err); + CheckNoError(err); + + // the db_identity after the checkpoint is the same + size_t after_db_id_len = 0; + char* after_db_id = rocksdb_get_db_identity(db, &after_db_id_len); + CheckCondition(after_db_id_len == 36); + CheckCondition(memcmp(after_db_id, before_db_id, after_db_id_len) == 0); + Free(&before_db_id); + Free(&after_db_id); + + rocksdb_close(db); + rocksdb_destroy_db(options_dbid_in_manifest, dbcheckpointname, &err); + CheckNoError(err); + rocksdb_options_destroy(options_dbid_in_manifest); + options_dbid_in_manifest = NULL; + + // re-open the default database + rocksdb_options_set_error_if_exists(options, 0); + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + rocksdb_options_set_error_if_exists(options, 1); + } + StartPhase("compactall"); rocksdb_compact_range(db, NULL, 0, NULL, 0); CheckGet(db, roptions, "foo", "hello"); diff --git a/include/rocksdb/c.h b/include/rocksdb/c.h index 89d8a2b18..ed1d871f2 100644 --- a/include/rocksdb/c.h +++ b/include/rocksdb/c.h @@ -505,6 +505,13 @@ extern ROCKSDB_LIBRARY_API char* rocksdb_get_cf_with_ts( rocksdb_column_family_handle_t* column_family, const char* key, size_t keylen, size_t* vallen, char** ts, size_t* tslen, char** errptr); +/** + * Returns a malloc() buffer with the DB identity, assigning the length to + * *id_len. Returns NULL if an error occurred. + */ +extern ROCKSDB_LIBRARY_API char* rocksdb_get_db_identity(rocksdb_t* db, + size_t* id_len); + // if values_list[i] == NULL and errs[i] == NULL, // then we got status.IsNotFound(), which we will not return. // all errors except status status.ok() and status.IsNotFound() are returned. @@ -1608,6 +1615,11 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_plain_table_factory( rocksdb_options_t*, uint32_t, int, double, size_t, size_t, char, unsigned char, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_write_dbid_to_manifest(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_write_dbid_to_manifest( + rocksdb_options_t*, unsigned char); + extern ROCKSDB_LIBRARY_API void rocksdb_options_set_min_level_to_compress( rocksdb_options_t* opt, int level); |