More Makefile Cleanup (#7097)

Summary:
Cleans up some of the dependencies on test code in the Makefile while building tools:
- Moves the test::RandomString, DBBaseTest::RandomString into Random
- Moves the test::RandomHumanReadableString into Random
- Moves the DestroyDir method into file_utils
- Moves the SetupSyncPointsToMockDirectIO into sync_point.
- Moves the FaultInjection Env and FS classes under env

These changes allow all of the tools to build without dependencies on test_util, thereby simplifying the build dependencies.  By moving the FaultInjection code, the dependency in db_stress on different libraries for debug vs release was eliminated.

Tested both release and debug builds via Make and CMake for both static and shared libraries.

More work remains to clean up how the tools are built and remove some unnecessary dependencies.  There is also more work that should be done to get the Makefile and CMake to align in their builds -- what is in the libraries and the sizes of the executables are different.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7097

Reviewed By: riversand963

Differential Revision: D22463160

Pulled By: pdillinger

fbshipit-source-id: e19462b53324ab3f0b7c72459dbc73165cc382b2

6 files changed, 35 insertions, 51 deletions

diff --git a/table/block_based/block_based_table_reader_test.cc b/table/block_based/block_based_table_reader_test.cc
index 6062b3d4e..e46e3de11 100644
--- a/table/block_based/block_based_table_reader_test.cc
+++ b/table/block_based/block_based_table_reader_test.cc
@@ -4,18 +4,20 @@
 //  (found in the LICENSE.Apache file in the root directory).
 
 #include "table/block_based/block_based_table_reader.h"
-#include "rocksdb/file_system.h"
-#include "table/block_based/partitioned_index_iterator.h"
 
 #include "db/table_properties_collector.h"
+#include "file/file_util.h"
 #include "options/options_helper.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
+#include "rocksdb/file_system.h"
 #include "table/block_based/block_based_table_builder.h"
 #include "table/block_based/block_based_table_factory.h"
+#include "table/block_based/partitioned_index_iterator.h"
 #include "table/format.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
+#include "util/random.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -33,7 +35,7 @@ class BlockBasedTableReaderTest
     std::tie(compression_type_, use_direct_reads_, index_type, no_block_cache) =
         GetParam();
 
-    test::SetupSyncPointsToMockDirectIO();
+    SetupSyncPointsToMockDirectIO();
     test_dir_ = test::PerThreadDBPath("block_based_table_reader_test");
     env_ = Env::Default();
     fs_ = FileSystem::Default();
@@ -46,7 +48,7 @@ class BlockBasedTableReaderTest
         static_cast<BlockBasedTableFactory*>(NewBlockBasedTableFactory(opts)));
   }
 
-  void TearDown() override { EXPECT_OK(test::DestroyDir(env_, test_dir_)); }
+  void TearDown() override { EXPECT_OK(DestroyDir(env_, test_dir_)); }
 
   // Creates a table with the specificied key value pairs (kv).
   void CreateTable(const std::string& table_name,
@@ -159,9 +161,9 @@ TEST_P(BlockBasedTableReaderTest, MultiGet) {
         sprintf(k, "%08u", key);
         std::string v;
         if (block % 2) {
-          v = test::RandomHumanReadableString(&rnd, 256);
+          v = rnd.HumanReadableString(256);
         } else {
-          test::RandomString(&rnd, 256, &v);
+          v = rnd.RandomString(256);
         }
         kv[std::string(k)] = v;
         key++;
@@ -256,8 +258,7 @@ TEST_P(BlockBasedTableReaderTestVerifyChecksum, ChecksumMismatch) {
         // and internal key size is required to be >= 8 bytes,
         // so use %08u as the format string.
         sprintf(k, "%08u", key);
-        std::string v;
-        test::RandomString(&rnd, 256, &v);
+        std::string v = rnd.RandomString(256);
         kv[std::string(k)] = v;
         key++;
       }
diff --git a/table/block_based/block_test.cc b/table/block_based/block_test.cc
index 7521c1515..c9f12d54e 100644
--- a/table/block_based/block_test.cc
+++ b/table/block_based/block_test.cc
@@ -29,12 +29,6 @@
 
 namespace ROCKSDB_NAMESPACE {
 
-static std::string RandomString(Random *rnd, int len) {
-  std::string r;
-  test::RandomString(rnd, len, &r);
-  return r;
-}
-
 std::string GenerateInternalKey(int primary_key, int secondary_key,
                                 int padding_size, Random *rnd) {
   char buf[50];
@@ -42,7 +36,7 @@ std::string GenerateInternalKey(int primary_key, int secondary_key,
   snprintf(buf, sizeof(buf), "%6d%4d", primary_key, secondary_key);
   std::string k(p);
   if (padding_size) {
-    k += RandomString(rnd, padding_size);
+    k += rnd->RandomString(padding_size);
   }
   AppendInternalKeyFooter(&k, 0 /* seqno */, kTypeValue);
 
@@ -67,7 +61,7 @@ void GenerateRandomKVs(std::vector<std::string> *keys,
       keys->emplace_back(GenerateInternalKey(i, j, padding_size, &rnd));
 
       // 100 bytes values
-      values->emplace_back(RandomString(&rnd, 100));
+      values->emplace_back(rnd.RandomString(100));
     }
   }
 }
diff --git a/table/block_based/data_block_hash_index_test.cc b/table/block_based/data_block_hash_index_test.cc
index 409a5bdc2..94fa7e94f 100644
--- a/table/block_based/data_block_hash_index_test.cc
+++ b/table/block_based/data_block_hash_index_test.cc
@@ -3,6 +3,8 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
+#include "table/block_based/data_block_hash_index.h"
+
 #include <cstdlib>
 #include <string>
 #include <unordered_map>
@@ -12,11 +14,11 @@
 #include "table/block_based/block.h"
 #include "table/block_based/block_based_table_reader.h"
 #include "table/block_based/block_builder.h"
-#include "table/block_based/data_block_hash_index.h"
 #include "table/get_context.h"
 #include "table/table_builder.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
+#include "util/random.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -35,12 +37,6 @@ bool SearchForOffset(DataBlockHashIndex& index, const char* data,
   return entry == restart_point;
 }
 
-// Random KV generator similer to block_test
-static std::string RandomString(Random* rnd, int len) {
-  std::string r;
-  test::RandomString(rnd, len, &r);
-  return r;
-}
 std::string GenerateKey(int primary_key, int secondary_key, int padding_size,
                         Random* rnd) {
   char buf[50];
@@ -48,7 +44,7 @@ std::string GenerateKey(int primary_key, int secondary_key, int padding_size,
   snprintf(buf, sizeof(buf), "%6d%4d", primary_key, secondary_key);
   std::string k(p);
   if (padding_size) {
-    k += RandomString(rnd, padding_size);
+    k += rnd->RandomString(padding_size);
   }
 
   return k;
@@ -71,7 +67,7 @@ void GenerateRandomKVs(std::vector<std::string>* keys,
       keys->emplace_back(GenerateKey(i, j, padding_size, &rnd));
 
       // 100 bytes values
-      values->emplace_back(RandomString(&rnd, 100));
+      values->emplace_back(rnd.RandomString(100));
     }
   }
 }
diff --git a/table/block_fetcher_test.cc b/table/block_fetcher_test.cc
index 6d1cd5ddd..e3c17b292 100644
--- a/table/block_fetcher_test.cc
+++ b/table/block_fetcher_test.cc
@@ -4,7 +4,10 @@
 //  (found in the LICENSE.Apache file in the root directory).
 
 #include "table/block_fetcher.h"
+
 #include "db/table_properties_collector.h"
+#include "env/composite_env_wrapper.h"
+#include "file/file_util.h"
 #include "options/options_helper.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
@@ -14,7 +17,6 @@
 #include "table/block_based/block_based_table_reader.h"
 #include "table/format.h"
 #include "test_util/testharness.h"
-#include "test_util/testutil.h"
 
 namespace ROCKSDB_NAMESPACE {
 namespace {
@@ -71,14 +73,14 @@ class BlockFetcherTest : public testing::Test {
 
  protected:
   void SetUp() override {
-    test::SetupSyncPointsToMockDirectIO();
+    SetupSyncPointsToMockDirectIO();
     test_dir_ = test::PerThreadDBPath("block_fetcher_test");
     env_ = Env::Default();
     fs_ = FileSystem::Default();
     ASSERT_OK(fs_->CreateDir(test_dir_, IOOptions(), nullptr));
   }
 
-  void TearDown() override { EXPECT_OK(test::DestroyDir(env_, test_dir_)); }
+  void TearDown() override { EXPECT_OK(DestroyDir(env_, test_dir_)); }
 
   void AssertSameBlock(const std::string& block1, const std::string& block2) {
     ASSERT_EQ(block1, block2);
diff --git a/table/merger_test.cc b/table/merger_test.cc
index 466e0eb42..13f225731 100644
--- a/table/merger_test.cc
+++ b/table/merger_test.cc
@@ -3,12 +3,13 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
-#include <vector>
 #include <string>
+#include <vector>
 
 #include "table/merging_iterator.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
+#include "util/random.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -24,7 +25,7 @@ class MergerTest : public testing::Test {
     std::vector<std::string> ret;
 
     for (size_t i = 0; i < len; ++i) {
-      InternalKey ik(test::RandomHumanReadableString(&rnd_, string_len), 0,
+      InternalKey ik(rnd_.HumanReadableString(string_len), 0,
                      ValueType::kTypeValue);
       ret.push_back(ik.Encode().ToString(false));
     }
@@ -44,8 +45,7 @@ class MergerTest : public testing::Test {
   }
 
   void SeekToRandom() {
-    InternalKey ik(test::RandomHumanReadableString(&rnd_, 5), 0,
-                   ValueType::kTypeValue);
+    InternalKey ik(rnd_.HumanReadableString(5), 0, ValueType::kTypeValue);
     Seek(ik.Encode().ToString(false));
   }
 
diff --git a/table/table_test.cc b/table/table_test.cc
index 7bc53a584..3554f3463 100644
--- a/table/table_test.cc
+++ b/table/table_test.cc
@@ -1227,8 +1227,7 @@ class FileChecksumTestHelper {
   void AddKVtoKVMap(int num_entries) {
     Random rnd(test::RandomSeed());
     for (int i = 0; i < num_entries; i++) {
-      std::string v;
-      test::RandomString(&rnd, 100, &v);
+      std::string v = rnd.RandomString(100);
       kv_map_[test::RandomKey(&rnd, 20)] = v;
     }
   }
@@ -1899,16 +1898,10 @@ TEST_P(BlockBasedTableTest, SkipPrefixBloomFilter) {
   }
 }
 
-static std::string RandomString(Random* rnd, int len) {
-  std::string r;
-  test::RandomString(rnd, len, &r);
-  return r;
-}
-
 void AddInternalKey(TableConstructor* c, const std::string& prefix,
                     std::string value = "v", int /*suffix_len*/ = 800) {
   static Random rnd(1023);
-  InternalKey k(prefix + RandomString(&rnd, 800), 0, kTypeValue);
+  InternalKey k(prefix + rnd.RandomString(800), 0, kTypeValue);
   c->Add(k.Encode().ToString(), value);
 }
 
@@ -2481,7 +2474,7 @@ TEST_P(BlockBasedTableTest, IndexSizeStat) {
   std::vector<std::string> keys;
 
   for (int i = 0; i < 100; ++i) {
-    keys.push_back(RandomString(&rnd, 10000));
+    keys.push_back(rnd.RandomString(10000));
   }
 
   // Each time we load one more key to the table. the table index block
@@ -2525,7 +2518,7 @@ TEST_P(BlockBasedTableTest, NumBlockStat) {
   for (int i = 0; i < 10; ++i) {
     // the key/val are slightly smaller than block size, so that each block
     // holds roughly one key/value pair.
-    c.Add(RandomString(&rnd, 900), "val");
+    c.Add(rnd.RandomString(900), "val");
   }
 
   std::vector<std::string> ks;
@@ -3607,9 +3600,8 @@ TEST_P(ParameterizedHarnessTest, RandomizedHarnessTest) {
   for (int num_entries = 0; num_entries < 2000;
        num_entries += (num_entries < 50 ? 1 : 200)) {
     for (int e = 0; e < num_entries; e++) {
-      std::string v;
       Add(test::RandomKey(&rnd, rnd.Skewed(4)),
-          test::RandomString(&rnd, rnd.Skewed(5), &v).ToString());
+          rnd.RandomString(rnd.Skewed(5)));
     }
     Test(&rnd);
   }
@@ -3621,8 +3613,7 @@ TEST_F(DBHarnessTest, RandomizedLongDB) {
   int num_entries = 100000;
   for (int e = 0; e < num_entries; e++) {
     std::string v;
-    Add(test::RandomKey(&rnd, rnd.Skewed(4)),
-        test::RandomString(&rnd, rnd.Skewed(5), &v).ToString());
+    Add(test::RandomKey(&rnd, rnd.Skewed(4)), rnd.RandomString(rnd.Skewed(5)));
   }
   Test(&rnd);
 
@@ -3878,8 +3869,8 @@ TEST_P(IndexBlockRestartIntervalTest, IndexBlockRestartInterval) {
   TableConstructor c(BytewiseComparator());
   static Random rnd(301);
   for (int i = 0; i < kKeysInTable; i++) {
-    InternalKey k(RandomString(&rnd, kKeySize), 0, kTypeValue);
-    c.Add(k.Encode().ToString(), RandomString(&rnd, kValSize));
+    InternalKey k(rnd.RandomString(kKeySize), 0, kTypeValue);
+    c.Add(k.Encode().ToString(), rnd.RandomString(kValSize));
   }
 
   std::vector<std::string> keys;
@@ -4541,9 +4532,9 @@ TEST_P(BlockBasedTableTest, DataBlockHashIndex) {
   static Random rnd(1048);
   for (int i = 0; i < kNumKeys; i++) {
     // padding one "0" to mark existent keys.
-    std::string random_key(RandomString(&rnd, kKeySize - 1) + "1");
+    std::string random_key(rnd.RandomString(kKeySize - 1) + "1");
     InternalKey k(random_key, 0, kTypeValue);
-    c.Add(k.Encode().ToString(), RandomString(&rnd, kValSize));
+    c.Add(k.Encode().ToString(), rnd.RandomString(kValSize));
   }
 
   std::vector<std::string> keys;