diff options
author | Yanqin Jin <yanqin@fb.com> | 2019-08-02 10:40:32 -0700 |
---|---|---|
committer | Facebook Github Bot <facebook-github-bot@users.noreply.github.com> | 2019-08-02 10:55:17 -0700 |
commit | 30edf1874c11762a6cacf4434112ce34d13100d3 (patch) | |
tree | 626d24446a435ace5cf191c65838a7247a24192a /buckifier | |
parent | d1c9ede1956a29472fbe7202cd3e8ee7aefa7c31 (diff) |
Change buckifier to support parameterized dependencies (#5648)
Summary:
Users may desire to specify extra dependencies via buck. This PR allows users to pass additional dependencies as a JSON object so that the buckifier script can generate TARGETS file with desired extra dependencies.
Test plan (on dev server)
```
$python buckifier/buckify_rocksdb.py '{"fake": {"extra_deps": [":test_dep", "//fakes/module:mock1"], "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"]}}'
Generating TARGETS
Extra dependencies:
{'': {'extra_compiler_flags': [], 'extra_deps': []}, 'test_dep1': {'extra_compiler_flags': ['-O2', '-DROCKSDB_LITE'], 'extra_deps': [':fake', '//dep1/mock']}}
Generated TARGETS Summary:
- 5 libs
- 0 binarys
- 296 tests
```
Verify the TARGETS file.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5648
Differential Revision: D16565043
Pulled By: riversand963
fbshipit-source-id: a6ef02274174fcf159692d7b846e828454d01e89
Diffstat (limited to 'buckifier')
-rw-r--r-- | buckifier/buckify_rocksdb.py | 97 | ||||
-rw-r--r-- | buckifier/targets_builder.py | 11 | ||||
-rw-r--r-- | buckifier/targets_cfg.py | 8 |
3 files changed, 93 insertions, 23 deletions
diff --git a/buckifier/buckify_rocksdb.py b/buckifier/buckify_rocksdb.py index 94b63a4e8..fc59cf583 100644 --- a/buckifier/buckify_rocksdb.py +++ b/buckifier/buckify_rocksdb.py @@ -4,12 +4,31 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals from targets_builder import TARGETSBuilder +import json import os import fnmatch import sys from util import ColorString +# This script generates TARGETS file for Buck. +# Buck is a build tool specifying dependencies among different build targets. +# User can pass extra dependencies as a JSON object via command line, and this +# script can include these dependencies in the generate TARGETS file. +# Usage: +# $python buckifier/buckify_rocksdb.py +# (This generates a TARGET file without user-specified dependency for unit +# tests.) +# $python buckifier/buckify_rocksdb.py \ +# '{"fake": { \ +# "extra_deps": [":test_dep", "//fakes/module:mock1"], \ +# "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"], \ +# } \ +# }' +# (Generated TARGETS file has test_dep and mock1 as dependencies for RocksDB +# unit tests, and will use the extra_compiler_flags to compile the unit test +# source.) + # tests to export as libraries for inclusion in other projects _EXPORTED_TEST_LIBS = ["env_basic_test"] @@ -86,8 +105,38 @@ def get_tests(repo_path): return tests +# Parse extra dependencies passed by user from command line +def get_dependencies(): + deps_map = { + ''.encode('ascii'): { + 'extra_deps'.encode('ascii'): [], + 'extra_compiler_flags'.encode('ascii'): [] + } + } + if len(sys.argv) < 2: + return deps_map + + def encode_dict(data): + rv = {} + for k, v in data.items(): + if isinstance(k, unicode): + k = k.encode('ascii') + if isinstance(v, unicode): + v = v.encode('ascii') + elif isinstance(v, list): + v = [x.encode('ascii') for x in v] + elif isinstance(v, dict): + v = encode_dict(v) + rv[k] = v + return rv + extra_deps = json.loads(sys.argv[1], object_hook=encode_dict) + for target_alias, deps in extra_deps.items(): + deps_map[target_alias] = deps + return deps_map + + # Prepare TARGETS file for buck -def generate_targets(repo_path): +def generate_targets(repo_path, deps_map): print(ColorString.info("Generating TARGETS")) # parsed src.mk file src_mk = parse_src_mk(repo_path) @@ -121,24 +170,33 @@ def generate_targets(repo_path): ["test_util/testutil.cc"], [":rocksdb_lib"]) + print("Extra dependencies:\n{0}".format(str(deps_map))) # test for every test we found in the Makefile - for test in sorted(tests): - match_src = [src for src in cc_files if ("/%s.c" % test) in src] - if len(match_src) == 0: - print(ColorString.warning("Cannot find .cc file for %s" % test)) - continue - elif len(match_src) > 1: - print(ColorString.warning("Found more than one .cc for %s" % test)) - print(match_src) - continue - - assert(len(match_src) == 1) - is_parallel = tests[test] - TARGETS.register_test(test, match_src[0], is_parallel) - - if test in _EXPORTED_TEST_LIBS: - test_library = "%s_lib" % test - TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"]) + for target_alias, deps in deps_map.items(): + for test in sorted(tests): + match_src = [src for src in cc_files if ("/%s.c" % test) in src] + if len(match_src) == 0: + print(ColorString.warning("Cannot find .cc file for %s" % test)) + continue + elif len(match_src) > 1: + print(ColorString.warning("Found more than one .cc for %s" % test)) + print(match_src) + continue + + assert(len(match_src) == 1) + is_parallel = tests[test] + test_target_name = \ + test if not target_alias else test + "_" + target_alias + TARGETS.register_test( + test_target_name, + match_src[0], + is_parallel, + deps['extra_deps'], + deps['extra_compiler_flags']) + + if test in _EXPORTED_TEST_LIBS: + test_library = "%s_lib" % test_target_name + TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"]) TARGETS.flush_tests() print(ColorString.info("Generated TARGETS Summary:")) @@ -163,8 +221,9 @@ def exit_with_error(msg): def main(): + deps_map = get_dependencies() # Generate TARGETS file for buck - ok = generate_targets(get_rocksdb_path()) + ok = generate_targets(get_rocksdb_path(), deps_map) if not ok: exit_with_error("Failed to generate TARGETS files") diff --git a/buckifier/targets_builder.py b/buckifier/targets_builder.py index 493cd8a8a..78db6a169 100644 --- a/buckifier/targets_builder.py +++ b/buckifier/targets_builder.py @@ -51,14 +51,21 @@ class TARGETSBuilder: pretty_list(deps))) self.total_bin = self.total_bin + 1 - def register_test(self, test_name, src, is_parallel): + def register_test(self, + test_name, + src, + is_parallel, + extra_deps, + extra_compiler_flags): exec_mode = "serial" if is_parallel: exec_mode = "parallel" self.tests_cfg += targets_cfg.test_cfg_template % ( test_name, str(src), - str(exec_mode)) + str(exec_mode), + extra_deps, + extra_compiler_flags) self.total_test = self.total_test + 1 diff --git a/buckifier/targets_cfg.py b/buckifier/targets_cfg.py index 0ebd6d942..19ea77727 100644 --- a/buckifier/targets_cfg.py +++ b/buckifier/targets_cfg.py @@ -140,11 +140,13 @@ test_cfg_template = """ [ "%s", "%s", "%s", + %s, + %s, ], """ unittests_template = """ -# [test_name, test_src, test_type] +# [test_name, test_src, test_type, extra_deps, extra_compiler_flags] ROCKS_TESTS = [ %s] @@ -153,6 +155,8 @@ ROCKS_TESTS = [ # will not be included. [ test_binary( + extra_compiler_flags = extra_compiler_flags, + extra_deps = extra_deps, parallelism = parallelism, rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS, rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS, @@ -163,7 +167,7 @@ ROCKS_TESTS = [ test_cc = test_cc, test_name = test_name, ) - for test_name, test_cc, parallelism in ROCKS_TESTS + for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS if not is_opt_mode ] """ |