Use Libraries in the RocksDB Makefile Build (#6660)

Summary:
Change the linking of tests/tools to be against a library rather than a list of objects.  This change substantially reduces the size of the objects produced.

peterd clean repo size: 264M
Before this change, with make all: 40G
After this change, with make all: 28G
With make LIB_MODE=shared all: 7.0G

The list of TESTS was changed from being hard-coded to generated from the test sources variable.  Note that there are some test sources that are not built as tests (though the set of tests is identical to the previous version).

Added OBJ_DIR option to Makefile to allow objects to be placed in an alternative location.  By default, OBJ_DIR is the same as before ("./").

This change is a precursor to being able to build/run the tests/tools linked against static libraries.  Additionally, it should be possible to clean up and merge some of the rules for building tests and the like if so desired.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6660

Reviewed By: riversand963

Differential Revision: D22244463

Pulled By: pdillinger

fbshipit-source-id: db9c6341d81ed62c2270374f4ede02fb9604c754

1 files changed, 53 insertions, 33 deletions

diff --git a/src.mk b/src.mk
index 3f92d6f89..c4fec346e 100644
--- a/src.mk
+++ b/src.mk
@@ -15,13 +15,13 @@ LIB_SOURCES =                                                   \
   db/c.cc                                                       \
   db/column_family.cc                                           \
   db/compacted_db_impl.cc                                       \
-  db/compaction/compaction.cc                                 	\
+  db/compaction/compaction.cc                                   \
   db/compaction/compaction_iterator.cc                          \
   db/compaction/compaction_job.cc                               \
   db/compaction/compaction_picker.cc                            \
   db/compaction/compaction_picker_fifo.cc                       \
   db/compaction/compaction_picker_level.cc                      \
-  db/compaction/compaction_picker_universal.cc                 	\
+  db/compaction/compaction_picker_universal.cc                  \
   db/convenience.cc                                             \
   db/db_filesnapshot.cc                                         \
   db/db_impl/db_impl.cc                                         \
@@ -36,7 +36,7 @@ LIB_SOURCES =                                                   \
   db/db_info_dumper.cc                                          \
   db/db_iter.cc                                                 \
   db/dbformat.cc                                                \
-  db/error_handler.cc						                                \
+  db/error_handler.cc                                           \
   db/event_helpers.cc                                           \
   db/experimental.cc                                            \
   db/external_sst_file_ingestion_job.cc                         \
@@ -77,7 +77,7 @@ LIB_SOURCES =                                                   \
   env/env_hdfs.cc                                               \
   env/env_posix.cc                                              \
   env/file_system.cc                                            \
-  env/fs_posix.cc                                           	  \
+  env/fs_posix.cc                                               \
   env/io_posix.cc                                               \
   env/mock_env.cc                                               \
   file/delete_scheduler.cc                                      \
@@ -150,7 +150,7 @@ LIB_SOURCES =                                                   \
   table/block_based/partitioned_index_reader.cc                 \
   table/block_based/reader_common.cc                            \
   table/block_based/uncompression_dict_reader.cc                \
-  table/block_fetcher.cc                             		        \
+  table/block_fetcher.cc                                        \
   table/cuckoo/cuckoo_table_builder.cc                          \
   table/cuckoo/cuckoo_table_factory.cc                          \
   table/cuckoo/cuckoo_table_reader.cc                           \
@@ -191,7 +191,7 @@ LIB_SOURCES =                                                   \
   util/random.cc                                                \
   util/rate_limiter.cc                                          \
   util/slice.cc                                                 \
-  util/file_checksum_helper.cc      				\
+  util/file_checksum_helper.cc                                  \
   util/status.cc                                                \
   util/string_util.cc                                           \
   util/thread_local.cc                                          \
@@ -216,7 +216,7 @@ LIB_SOURCES =                                                   \
   utilities/memory/memory_util.cc                               \
   utilities/merge_operators/max.cc                              \
   utilities/merge_operators/put.cc                              \
-  utilities/merge_operators/sortlist.cc                  		    \
+  utilities/merge_operators/sortlist.cc                         \
   utilities/merge_operators/string_append/stringappend.cc       \
   utilities/merge_operators/string_append/stringappend2.cc      \
   utilities/merge_operators/uint64add.cc                        \
@@ -277,7 +277,7 @@ ANALYZER_LIB_SOURCES =                                          \
 
 MOCK_LIB_SOURCES =                                              \
   table/mock_table.cc                                           \
-  test_util/fault_injection_test_fs.cc				\
+  test_util/fault_injection_test_fs.cc                          \
   test_util/fault_injection_test_env.cc
 
 BENCH_LIB_SOURCES =                                             \
@@ -307,10 +307,32 @@ FOLLY_SOURCES = \
   third-party/folly/folly/synchronization/ParkingLot.cpp                       \
   third-party/folly/folly/synchronization/WaitOptions.cpp                      \
 
-MAIN_SOURCES =                                                          \
+TOOLS_MAIN_SOURCES =                                                    \
+  db_stress_tool/db_stress.cc                                           \
+  tools/blob_dump.cc                                                    \
+  tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc         \
+  tools/db_repl_stress.cc                                               \
+  tools/db_sanity_test.cc                                               \
+  tools/ldb.cc                                                          \
+  tools/sst_dump.cc                                                     \
+  tools/write_stress.cc                                                 \
+  tools/dump/rocksdb_dump.cc                                            \
+  tools/dump/rocksdb_undump.cc                                          \
+  tools/trace_analyzer.cc                                               \
+
+BENCH_MAIN_SOURCES =                                                    \
   cache/cache_bench.cc                                                  \
+  db/range_del_aggregator_bench.cc                                      \
+  memtable/memtablerep_bench.cc                                         \
+  table/table_reader_bench.cc                                           \
+  tools/db_bench.cc                                                     \
+  util/filter_bench.cc                                                  \
+  utilities/persistent_cache/persistent_cache_bench.cc                  \
+  #util/log_write_bench.cc                                               \
+
+TEST_MAIN_SOURCES =                                                     \
   cache/cache_test.cc                                                   \
-  db_stress_tool/db_stress.cc                                           \
+  cache/lru_cache_test.cc                                               \
   db/blob/blob_file_addition_test.cc                                    \
   db/blob/blob_file_garbage_test.cc                                     \
   db/blob/db_blob_index_test.cc                                         \
@@ -324,7 +346,7 @@ MAIN_SOURCES =                                                          \
   db/corruption_test.cc                                                 \
   db/cuckoo_table_db_test.cc                                            \
   db/db_basic_test.cc                                                   \
-  db/db_with_timestamp_basic_test.cc					\
+  db/db_with_timestamp_basic_test.cc                                    \
   db/db_block_cache_test.cc                                             \
   db/db_bloom_filter_test.cc                                            \
   db/db_compaction_filter_test.cc                                       \
@@ -332,6 +354,7 @@ MAIN_SOURCES =                                                          \
   db/db_dynamic_level_test.cc                                           \
   db/db_encryption_test.cc                                              \
   db/db_flush_test.cc                                                   \
+  db/import_column_family_test.cc                                       \
   db/db_inplace_update_test.cc                                          \
   db/db_io_failure_test.cc                                              \
   db/db_iter_test.cc                                                    \
@@ -340,7 +363,7 @@ MAIN_SOURCES =                                                          \
   db/db_log_iter_test.cc                                                \
   db/db_memtable_test.cc                                                \
   db/db_merge_operator_test.cc                                          \
-  db/db_merge_operand_test.cc                                          	\
+  db/db_merge_operand_test.cc                                           \
   db/db_options_test.cc                                                 \
   db/db_properties_test.cc                                              \
   db/db_range_del_test.cc                                               \
@@ -358,8 +381,7 @@ MAIN_SOURCES =                                                          \
   db/db_write_test.cc                                                   \
   db/dbformat_test.cc                                                   \
   db/deletefile_test.cc                                                 \
-  db/env_timed_test.cc                                                  \
-  db/error_handler_fs_test.cc						\
+  db/error_handler_fs_test.cc                                           \
   db/external_sst_file_basic_test.cc                                    \
   db/external_sst_file_test.cc                                          \
   db/fault_injection_test.cc                                            \
@@ -367,29 +389,21 @@ MAIN_SOURCES =                                                          \
   db/file_reader_writer_test.cc                                         \
   db/filename_test.cc                                                   \
   db/flush_job_test.cc                                                  \
-  db/hash_table_test.cc                                                 \
-  db/hash_test.cc                                                       \
-  db/heap_test.cc                                                       \
   db/listener_test.cc                                                   \
   db/log_test.cc                                                        \
-  db/lru_cache_test.cc                                                  \
   db/manual_compaction_test.cc                                          \
   db/memtable_list_test.cc                                              \
   db/merge_helper_test.cc                                               \
   db/merge_test.cc                                                      \
-  db/obsolete_files_test.cc						                                  \
-  db/options_settable_test.cc                                           \
+  db/obsolete_files_test.cc                                             \
   db/options_file_test.cc                                               \
   db/perf_context_test.cc                                               \
-  db/persistent_cache_test.cc                                           \
   db/plain_table_db_test.cc                                             \
   db/prefix_test.cc                                                     \
   db/repair_test.cc                                                     \
   db/range_del_aggregator_test.cc                                       \
-  db/range_del_aggregator_bench.cc                                      \
   db/range_tombstone_fragmenter_test.cc                                 \
   db/table_properties_collector_test.cc                                 \
-  db/util_merge_operators_test.cc                                       \
   db/version_builder_test.cc                                            \
   db/version_edit_test.cc                                               \
   db/version_set_test.cc                                                \
@@ -401,6 +415,7 @@ MAIN_SOURCES =                                                          \
   env/env_test.cc                                                       \
   env/io_posix_test.cc                                                  \
   env/mock_env_test.cc                                                  \
+  file/delete_scheduler_test.cc                                         \
   file/random_access_file_reader_test.cc                                \
   logging/auto_roll_logger_test.cc                                      \
   logging/env_logger_test.cc                                            \
@@ -408,13 +423,13 @@ MAIN_SOURCES =                                                          \
   memory/arena_test.cc                                                  \
   memory/memkind_kmem_allocator_test.cc                                 \
   memtable/inlineskiplist_test.cc                                       \
-  memtable/memtablerep_bench.cc                                         \
   memtable/skiplist_test.cc                                             \
   memtable/write_buffer_manager_test.cc                                 \
   monitoring/histogram_test.cc                                          \
   monitoring/iostats_context_test.cc                                    \
   monitoring/statistics_test.cc                                         \
   monitoring/stats_history_test.cc                                      \
+  options/options_settable_test.cc                                      \
   options/options_test.cc                                               \
   table/block_based/block_based_filter_block_test.cc                    \
   table/block_based/block_based_table_reader_test.cc                    \
@@ -427,19 +442,14 @@ MAIN_SOURCES =                                                          \
   table/cuckoo/cuckoo_table_reader_test.cc                              \
   table/merger_test.cc                                                  \
   table/sst_file_reader_test.cc                                         \
-  table/table_reader_bench.cc                                           \
   table/table_test.cc                                                   \
   table/block_fetcher_test.cc                                           \
-  third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc                  \
+  test_util/testutil_test.cc                                            \
   tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc         \
-  tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc         \
-  tools/db_bench.cc                                                     \
-  tools/db_bench_tool_test.cc                                           \
-  tools/db_sanity_test.cc                                               \
   tools/ldb_cmd_test.cc                                                 \
   tools/reduce_levels_test.cc                                           \
   tools/sst_dump_test.cc                                                \
-  tools/trace_analyzer_test.cc				                \
+  tools/trace_analyzer_test.cc                                          \
   trace_replay/block_cache_tracer_test.cc                               \
   trace_replay/io_tracer_test.cc                                        \
   util/autovector_test.cc                                               \
@@ -449,9 +459,11 @@ MAIN_SOURCES =                                                          \
   util/defer_test.cc                                                    \
   util/dynamic_bloom_test.cc                                            \
   util/filelock_test.cc                                                 \
-  util/log_write_bench.cc                                               \
-  util/rate_limiter_test.cc                                             \
+  util/file_reader_writer_test.cc                                       \
+  util/hash_test.cc                                                     \
+  util/heap_test.cc                                                     \
   util/random_test.cc                                                   \
+  util/rate_limiter_test.cc                                             \
   util/repeatable_thread_test.cc                                        \
   util/slice_test.cc                                                    \
   util/slice_transform_test.cc                                          \
@@ -467,11 +479,14 @@ MAIN_SOURCES =                                                          \
   utilities/cassandra/cassandra_row_merge_test.cc                       \
   utilities/cassandra/cassandra_serialize_test.cc                       \
   utilities/checkpoint/checkpoint_test.cc                               \
+  utilities/env_timed_test.cc                                           \
   utilities/memory/memory_test.cc                                       \
   utilities/merge_operators/string_append/stringappend_test.cc          \
   utilities/object_registry_test.cc                                     \
   utilities/option_change_migration/option_change_migration_test.cc     \
   utilities/options/options_util_test.cc                                \
+  utilities/persistent_cache/hash_table_test.cc                         \
+  utilities/persistent_cache/persistent_cache_test.cc                   \
   utilities/simulator_cache/cache_simulator_test.cc                     \
   utilities/simulator_cache/sim_cache_test.cc                           \
   utilities/table_properties_collectors/compact_on_deletion_collector_test.cc  \
@@ -481,8 +496,13 @@ MAIN_SOURCES =                                                          \
   utilities/transactions/write_prepared_transaction_test.cc             \
   utilities/transactions/write_unprepared_transaction_test.cc           \
   utilities/ttl/ttl_test.cc                                             \
+  utilities/util_merge_operators_test.cc                                \
   utilities/write_batch_with_index/write_batch_with_index_test.cc       \
 
+TEST_MAIN_SOURCES_C = \
+  db/c_test.c                                                           \
+
+
 JNI_NATIVE_SOURCES =                                          \
   java/rocksjni/backupenginejni.cc                            \
   java/rocksjni/backupablejni.cc                              \