1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2013 The Chromium OS Authors. 3# 4 5"""Control module for buildman 6 7This holds the main control logic for buildman, when not running tests. 8""" 9 10import multiprocessing 11import os 12import shutil 13import sys 14 15from buildman import boards 16from buildman import bsettings 17from buildman import cfgutil 18from buildman import toolchain 19from buildman.builder import Builder 20from patman import gitutil 21from patman import patchstream 22from u_boot_pylib import command 23from u_boot_pylib import terminal 24from u_boot_pylib.terminal import tprint 25 26TEST_BUILDER = None 27 28def get_plural(count): 29 """Returns a plural 's' if count is not 1""" 30 return 's' if count != 1 else '' 31 32 33def count_build_commits(commits, step): 34 """Calculate the number of commits to be built 35 36 Args: 37 commits (list of Commit): Commits to build or None 38 step (int): Step value for commits, typically 1 39 40 Returns: 41 Number of commits that will be built 42 """ 43 if commits: 44 count = len(commits) 45 return (count + step - 1) // step 46 return 0 47 48 49def get_action_summary(is_summary, commit_count, selected, threads, jobs): 50 """Return a string summarising the intended action. 51 52 Args: 53 is_summary (bool): True if this is a summary (otherwise it is building) 54 commits (list): List of commits being built 55 selected (list of Board): List of Board objects that are marked 56 step (int): Step increment through commits 57 threads (int): Number of processor threads being used 58 jobs (int): Number of jobs to build at once 59 60 Returns: 61 Summary string. 62 """ 63 if commit_count: 64 commit_str = f'{commit_count} commit{get_plural(commit_count)}' 65 else: 66 commit_str = 'current source' 67 msg = (f"{'Summary of' if is_summary else 'Building'} " 68 f'{commit_str} for {len(selected)} boards') 69 msg += (f' ({threads} thread{get_plural(threads)}, ' 70 f'{jobs} job{get_plural(jobs)} per thread)') 71 return msg 72 73# pylint: disable=R0913 74def show_actions(series, why_selected, boards_selected, output_dir, 75 board_warnings, step, threads, jobs, verbose): 76 """Display a list of actions that we would take, if not a dry run. 77 78 Args: 79 series: Series object 80 why_selected: Dictionary where each key is a buildman argument 81 provided by the user, and the value is the list of boards 82 brought in by that argument. For example, 'arm' might bring 83 in 400 boards, so in this case the key would be 'arm' and 84 the value would be a list of board names. 85 boards_selected: Dict of selected boards, key is target name, 86 value is Board object 87 output_dir (str): Output directory for builder 88 board_warnings: List of warnings obtained from board selected 89 step (int): Step increment through commits 90 threads (int): Number of processor threads being used 91 jobs (int): Number of jobs to build at once 92 verbose (bool): True to indicate why each board was selected 93 """ 94 col = terminal.Color() 95 print('Dry run, so not doing much. But I would do this:') 96 print() 97 if series: 98 commits = series.commits 99 else: 100 commits = None 101 print(get_action_summary(False, count_build_commits(commits, step), 102 boards_selected, threads, jobs)) 103 print(f'Build directory: {output_dir}') 104 if commits: 105 for upto in range(0, len(series.commits), step): 106 commit = series.commits[upto] 107 print(' ', col.build(col.YELLOW, commit.hash[:8], bright=False), end=' ') 108 print(commit.subject) 109 print() 110 for arg in why_selected: 111 if arg != 'all': 112 print(arg, f': {len(why_selected[arg])} boards') 113 if verbose: 114 print(f" {' '.join(why_selected[arg])}") 115 print('Total boards to build for each ' 116 f"commit: {len(why_selected['all'])}\n") 117 if board_warnings: 118 for warning in board_warnings: 119 print(col.build(col.YELLOW, warning)) 120 121def show_toolchain_prefix(brds, toolchains): 122 """Show information about a the tool chain used by one or more boards 123 124 The function checks that all boards use the same toolchain, then prints 125 the correct value for CROSS_COMPILE. 126 127 Args: 128 boards: Boards object containing selected boards 129 toolchains: Toolchains object containing available toolchains 130 131 Return: 132 None on success, string error message otherwise 133 """ 134 board_selected = brds.get_selected_dict() 135 tc_set = set() 136 for brd in board_selected.values(): 137 tc_set.add(toolchains.Select(brd.arch)) 138 if len(tc_set) != 1: 139 sys.exit('Supplied boards must share one toolchain') 140 tchain = tc_set.pop() 141 print(tchain.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) 142 143def show_arch(brds): 144 """Show information about a the architecture used by one or more boards 145 146 The function checks that all boards use the same architecture, then prints 147 the correct value for ARCH. 148 149 Args: 150 boards: Boards object containing selected boards 151 152 Return: 153 None on success, string error message otherwise 154 """ 155 board_selected = brds.get_selected_dict() 156 arch_set = set() 157 for brd in board_selected.values(): 158 arch_set.add(brd.arch) 159 if len(arch_set) != 1: 160 sys.exit('Supplied boards must share one arch') 161 print(arch_set.pop()) 162 163def get_allow_missing(opt_allow, opt_no_allow, num_selected, has_branch): 164 """Figure out whether to allow external blobs 165 166 Uses the allow-missing setting and the provided arguments to decide whether 167 missing external blobs should be allowed 168 169 Args: 170 opt_allow (bool): True if --allow-missing flag is set 171 opt_no_allow (bool): True if --no-allow-missing flag is set 172 num_selected (int): Number of selected board 173 has_branch (bool): True if a git branch (to build) has been provided 174 175 Returns: 176 bool: True to allow missing external blobs, False to produce an error if 177 external blobs are used 178 """ 179 allow_missing = False 180 am_setting = bsettings.get_global_item_value('allow-missing') 181 if am_setting: 182 if am_setting == 'always': 183 allow_missing = True 184 if 'multiple' in am_setting and num_selected > 1: 185 allow_missing = True 186 if 'branch' in am_setting and has_branch: 187 allow_missing = True 188 189 if opt_allow: 190 allow_missing = True 191 if opt_no_allow: 192 allow_missing = False 193 return allow_missing 194 195 196def count_commits(branch, count, col, git_dir): 197 """Could the number of commits in the branch/ranch being built 198 199 Args: 200 branch (str): Name of branch to build, or None if none 201 count (int): Number of commits to build, or -1 for all 202 col (Terminal.Color): Color object to use 203 git_dir (str): Git directory to use, e.g. './.git' 204 205 Returns: 206 tuple: 207 Number of commits being built 208 True if the 'branch' string contains a range rather than a simple 209 name 210 """ 211 has_range = branch and '..' in branch 212 if count == -1: 213 if not branch: 214 count = 1 215 else: 216 if has_range: 217 count, msg = gitutil.count_commits_in_range(git_dir, branch) 218 else: 219 count, msg = gitutil.count_commits_in_branch(git_dir, branch) 220 if count is None: 221 sys.exit(col.build(col.RED, msg)) 222 elif count == 0: 223 sys.exit(col.build(col.RED, 224 f"Range '{branch}' has no commits")) 225 if msg: 226 print(col.build(col.YELLOW, msg)) 227 count += 1 # Build upstream commit also 228 229 if not count: 230 msg = (f"No commits found to process in branch '{branch}': " 231 "set branch's upstream or use -c flag") 232 sys.exit(col.build(col.RED, msg)) 233 return count, has_range 234 235 236def determine_series(selected, col, git_dir, count, branch, work_in_output): 237 """Determine the series which is to be built, if any 238 239 If there is a series, the commits in that series are numbered by setting 240 their sequence value (starting from 0). This is used by tests. 241 242 Args: 243 selected (list of Board): List of Board objects that are marked 244 selected 245 col (Terminal.Color): Color object to use 246 git_dir (str): Git directory to use, e.g. './.git' 247 count (int): Number of commits in branch 248 branch (str): Name of branch to build, or None if none 249 work_in_output (bool): True to work in the output directory 250 251 Returns: 252 Series: Series to build, or None for none 253 254 Read the metadata from the commits. First look at the upstream commit, 255 then the ones in the branch. We would like to do something like 256 upstream/master~..branch but that isn't possible if upstream/master is 257 a merge commit (it will list all the commits that form part of the 258 merge) 259 260 Conflicting tags are not a problem for buildman, since it does not use 261 them. For example, Series-version is not useful for buildman. On the 262 other hand conflicting tags will cause an error. So allow later tags 263 to overwrite earlier ones by setting allow_overwrite=True 264 """ 265 266 # Work out how many commits to build. We want to build everything on the 267 # branch. We also build the upstream commit as a control so we can see 268 # problems introduced by the first commit on the branch. 269 count, has_range = count_commits(branch, count, col, git_dir) 270 if work_in_output: 271 if len(selected) != 1: 272 sys.exit(col.build(col.RED, 273 '-w can only be used with a single board')) 274 if count != 1: 275 sys.exit(col.build(col.RED, 276 '-w can only be used with a single commit')) 277 278 if branch: 279 if count == -1: 280 if has_range: 281 range_expr = branch 282 else: 283 range_expr = gitutil.get_range_in_branch(git_dir, branch) 284 upstream_commit = gitutil.get_upstream(git_dir, branch) 285 series = patchstream.get_metadata_for_list(upstream_commit, 286 git_dir, 1, series=None, allow_overwrite=True) 287 288 series = patchstream.get_metadata_for_list(range_expr, 289 git_dir, None, series, allow_overwrite=True) 290 else: 291 # Honour the count 292 series = patchstream.get_metadata_for_list(branch, 293 git_dir, count, series=None, allow_overwrite=True) 294 295 # Number the commits for test purposes 296 for i, commit in enumerate(series.commits): 297 commit.sequence = i 298 else: 299 series = None 300 return series 301 302 303def do_fetch_arch(toolchains, col, fetch_arch): 304 """Handle the --fetch-arch option 305 306 Args: 307 toolchains (Toolchains): Tool chains to use 308 col (terminal.Color): Color object to build 309 fetch_arch (str): Argument passed to the --fetch-arch option 310 311 Returns: 312 int: Return code for buildman 313 """ 314 if fetch_arch == 'list': 315 sorted_list = toolchains.ListArchs() 316 print(col.build( 317 col.BLUE, 318 f"Available architectures: {' '.join(sorted_list)}\n")) 319 return 0 320 321 if fetch_arch == 'all': 322 fetch_arch = ','.join(toolchains.ListArchs()) 323 print(col.build(col.CYAN, 324 f'\nDownloading toolchains: {fetch_arch}')) 325 for arch in fetch_arch.split(','): 326 print() 327 ret = toolchains.FetchAndInstall(arch) 328 if ret: 329 return ret 330 return 0 331 332 333def get_toolchains(toolchains, col, override_toolchain, fetch_arch, 334 list_tool_chains, verbose): 335 """Get toolchains object to use 336 337 Args: 338 toolchains (Toolchains or None): Toolchains to use. If None, then a 339 Toolchains object will be created and scanned 340 col (Terminal.Color): Color object 341 override_toolchain (str or None): Override value for toolchain, or None 342 fetch_arch (bool): True to fetch the toolchain for the architectures 343 list_tool_chains (bool): True to list all tool chains 344 verbose (bool): True for verbose output when listing toolchains 345 346 Returns: 347 Either: 348 int: Operation completed and buildman should exit with exit code 349 Toolchains: Toolchains object to use 350 """ 351 no_toolchains = toolchains is None 352 if no_toolchains: 353 toolchains = toolchain.Toolchains(override_toolchain) 354 355 if fetch_arch: 356 return do_fetch_arch(toolchains, col, fetch_arch) 357 358 if no_toolchains: 359 toolchains.GetSettings() 360 toolchains.Scan(list_tool_chains and verbose) 361 if list_tool_chains: 362 toolchains.List() 363 print() 364 return 0 365 return toolchains 366 367 368def get_boards_obj(output_dir, regen_board_list, maintainer_check, full_check, 369 threads, verbose): 370 """Object the Boards object to use 371 372 Creates the output directory and ensures there is a boards.cfg file, then 373 read it in. 374 375 Args: 376 output_dir (str): Output directory to use 377 regen_board_list (bool): True to just regenerate the board list 378 maintainer_check (bool): True to just run a maintainer check 379 full_check (bool): True to just run a full check of Kconfig and 380 maintainers 381 threads (int or None): Number of threads to use to create boards file 382 verbose (bool): False to suppress output from boards-file generation 383 384 Returns: 385 Either: 386 int: Operation completed and buildman should exit with exit code 387 Boards: Boards object to use 388 """ 389 brds = boards.Boards() 390 nr_cpus = threads or multiprocessing.cpu_count() 391 if maintainer_check or full_check: 392 warnings = brds.build_board_list(jobs=nr_cpus, 393 warn_targets=full_check)[1] 394 if warnings: 395 for warn in warnings: 396 print(warn, file=sys.stderr) 397 return 2 398 return 0 399 400 if not os.path.exists(output_dir): 401 os.makedirs(output_dir) 402 board_file = os.path.join(output_dir, 'boards.cfg') 403 if regen_board_list and regen_board_list != '-': 404 board_file = regen_board_list 405 406 okay = brds.ensure_board_list(board_file, nr_cpus, force=regen_board_list, 407 quiet=not verbose) 408 if regen_board_list: 409 return 0 if okay else 2 410 brds.read_boards(board_file) 411 return brds 412 413 414def determine_boards(brds, args, col, opt_boards, exclude_list): 415 """Determine which boards to build 416 417 Each element of args and exclude can refer to a board name, arch or SoC 418 419 Args: 420 brds (Boards): Boards object 421 args (list of str): Arguments describing boards to build 422 col (Terminal.Color): Color object 423 opt_boards (list of str): Specific boards to build, or None for all 424 exclude_list (list of str): Arguments describing boards to exclude 425 426 Returns: 427 tuple: 428 list of Board: List of Board objects that are marked selected 429 why_selected: Dictionary where each key is a buildman argument 430 provided by the user, and the value is the list of boards 431 brought in by that argument. For example, 'arm' might bring 432 in 400 boards, so in this case the key would be 'arm' and 433 the value would be a list of board names. 434 board_warnings: List of warnings obtained from board selected 435 """ 436 exclude = [] 437 if exclude_list: 438 for arg in exclude_list: 439 exclude += arg.split(',') 440 441 if opt_boards: 442 requested_boards = [] 443 for brd in opt_boards: 444 requested_boards += brd.split(',') 445 else: 446 requested_boards = None 447 why_selected, board_warnings = brds.select_boards(args, exclude, 448 requested_boards) 449 selected = brds.get_selected() 450 if not selected: 451 sys.exit(col.build(col.RED, 'No matching boards found')) 452 return selected, why_selected, board_warnings 453 454 455def adjust_args(args, series, selected): 456 """Adjust arguments according to various constraints 457 458 Updates verbose, show_errors, threads, jobs and step 459 460 Args: 461 args (Namespace): Namespace object to adjust 462 series (Series): Series being built / summarised 463 selected (list of Board): List of Board objects that are marked 464 """ 465 if not series and not args.dry_run: 466 args.verbose = True 467 if not args.summary: 468 args.show_errors = True 469 470 # By default we have one thread per CPU. But if there are not enough jobs 471 # we can have fewer threads and use a high '-j' value for make. 472 if args.threads is None: 473 args.threads = min(multiprocessing.cpu_count(), len(selected)) 474 if not args.jobs: 475 args.jobs = max(1, (multiprocessing.cpu_count() + 476 len(selected) - 1) // len(selected)) 477 478 if not args.step: 479 args.step = len(series.commits) - 1 480 481 # We can't show function sizes without board details at present 482 if args.show_bloat: 483 args.show_detail = True 484 485 486def setup_output_dir(output_dir, work_in_output, branch, no_subdirs, col, 487 clean_dir): 488 """Set up the output directory 489 490 Args: 491 output_dir (str): Output directory provided by the user, or None if none 492 work_in_output (bool): True to work in the output directory 493 branch (str): Name of branch to build, or None if none 494 no_subdirs (bool): True to put the output in the top-level output dir 495 clean_dir: Used for tests only, indicates that the existing output_dir 496 should be removed before starting the build 497 498 Returns: 499 str: Updated output directory pathname 500 """ 501 if not output_dir: 502 if work_in_output: 503 sys.exit(col.build(col.RED, '-w requires that you specify -o')) 504 output_dir = '..' 505 if branch and not no_subdirs: 506 # As a special case allow the board directory to be placed in the 507 # output directory itself rather than any subdirectory. 508 dirname = branch.replace('/', '_') 509 output_dir = os.path.join(output_dir, dirname) 510 if clean_dir and os.path.exists(output_dir): 511 shutil.rmtree(output_dir) 512 return output_dir 513 514 515def run_builder(builder, commits, board_selected, args): 516 """Run the builder or show the summary 517 518 Args: 519 commits (list of Commit): List of commits being built, None if no branch 520 boards_selected (dict): Dict of selected boards: 521 key: target name 522 value: Board object 523 args (Namespace): Namespace to use 524 525 Returns: 526 int: Return code for buildman 527 """ 528 gnu_make = command.output(os.path.join(args.git, 529 'scripts/show-gnu-make'), raise_on_error=False).rstrip() 530 if not gnu_make: 531 sys.exit('GNU Make not found') 532 builder.gnu_make = gnu_make 533 534 if not args.ide: 535 commit_count = count_build_commits(commits, args.step) 536 tprint(get_action_summary(args.summary, commit_count, board_selected, 537 args.threads, args.jobs)) 538 539 builder.set_display_options( 540 args.show_errors, args.show_sizes, args.show_detail, args.show_bloat, 541 args.list_error_boards, args.show_config, args.show_environment, 542 args.filter_dtb_warnings, args.filter_migration_warnings, args.ide) 543 if args.summary: 544 builder.show_summary(commits, board_selected) 545 else: 546 fail, warned, excs = builder.build_boards( 547 commits, board_selected, args.keep_outputs, args.verbose) 548 if excs: 549 return 102 550 if fail: 551 return 100 552 if warned and not args.ignore_warnings: 553 return 101 554 return 0 555 556 557def calc_adjust_cfg(adjust_cfg, reproducible_builds): 558 """Calculate the value to use for adjust_cfg 559 560 Args: 561 adjust_cfg (list of str): List of configuration changes. See cfgutil for 562 details 563 reproducible_builds (bool): True to adjust the configuration to get 564 reproduceable builds 565 566 Returns: 567 adjust_cfg (list of str): List of configuration changes 568 """ 569 adjust_cfg = cfgutil.convert_list_to_dict(adjust_cfg) 570 571 # Drop LOCALVERSION_AUTO since it changes the version string on every commit 572 if reproducible_builds: 573 # If these are mentioned, leave the local version alone 574 if 'LOCALVERSION' in adjust_cfg or 'LOCALVERSION_AUTO' in adjust_cfg: 575 print('Not dropping LOCALVERSION_AUTO for reproducible build') 576 else: 577 adjust_cfg['LOCALVERSION_AUTO'] = '~' 578 return adjust_cfg 579 580 581def do_buildman(args, toolchains=None, make_func=None, brds=None, 582 clean_dir=False, test_thread_exceptions=False): 583 """The main control code for buildman 584 585 Args: 586 args: ArgumentParser object 587 args: Command line arguments (list of strings) 588 toolchains: Toolchains to use - this should be a Toolchains() 589 object. If None, then it will be created and scanned 590 make_func: Make function to use for the builder. This is called 591 to execute 'make'. If this is None, the normal function 592 will be used, which calls the 'make' tool with suitable 593 arguments. This setting is useful for tests. 594 brds: Boards() object to use, containing a list of available 595 boards. If this is None it will be created and scanned. 596 clean_dir: Used for tests only, indicates that the existing output_dir 597 should be removed before starting the build 598 test_thread_exceptions: Uses for tests only, True to make the threads 599 raise an exception instead of reporting their result. This simulates 600 a failure in the code somewhere 601 """ 602 # Used so testing can obtain the builder: pylint: disable=W0603 603 global TEST_BUILDER 604 605 gitutil.setup() 606 col = terminal.Color() 607 608 git_dir = os.path.join(args.git, '.git') 609 610 toolchains = get_toolchains(toolchains, col, args.override_toolchain, 611 args.fetch_arch, args.list_tool_chains, 612 args.verbose) 613 if isinstance(toolchains, int): 614 return toolchains 615 616 output_dir = setup_output_dir( 617 args.output_dir, args.work_in_output, args.branch, 618 args.no_subdirs, col, clean_dir) 619 620 # Work out what subset of the boards we are building 621 if not brds: 622 brds = get_boards_obj(output_dir, args.regen_board_list, 623 args.maintainer_check, args.full_check, 624 args.threads, args.verbose and 625 not args.print_arch and not args.print_prefix) 626 if isinstance(brds, int): 627 return brds 628 629 selected, why_selected, board_warnings = determine_boards( 630 brds, args.terms, col, args.boards, args.exclude) 631 632 if args.print_prefix: 633 show_toolchain_prefix(brds, toolchains) 634 return 0 635 636 if args.print_arch: 637 show_arch(brds) 638 return 0 639 640 series = determine_series(selected, col, git_dir, args.count, 641 args.branch, args.work_in_output) 642 643 adjust_args(args, series, selected) 644 645 # For a dry run, just show our actions as a sanity check 646 if args.dry_run: 647 show_actions(series, why_selected, selected, output_dir, board_warnings, 648 args.step, args.threads, args.jobs, 649 args.verbose) 650 return 0 651 652 # Create a new builder with the selected args 653 builder = Builder(toolchains, output_dir, git_dir, 654 args.threads, args.jobs, checkout=True, 655 show_unknown=args.show_unknown, step=args.step, 656 no_subdirs=args.no_subdirs, full_path=args.full_path, 657 verbose_build=args.verbose_build, 658 mrproper=args.mrproper, 659 per_board_out_dir=args.per_board_out_dir, 660 config_only=args.config_only, 661 squash_config_y=not args.preserve_config_y, 662 warnings_as_errors=args.warnings_as_errors, 663 work_in_output=args.work_in_output, 664 test_thread_exceptions=test_thread_exceptions, 665 adjust_cfg=calc_adjust_cfg(args.adjust_cfg, 666 args.reproducible_builds), 667 allow_missing=get_allow_missing(args.allow_missing, 668 args.no_allow_missing, 669 len(selected), args.branch), 670 no_lto=args.no_lto, 671 reproducible_builds=args.reproducible_builds, 672 force_build = args.force_build, 673 force_build_failures = args.force_build_failures, 674 force_reconfig = args.force_reconfig, in_tree = args.in_tree, 675 force_config_on_failure=not args.quick, make_func=make_func) 676 677 TEST_BUILDER = builder 678 679 return run_builder(builder, series.commits if series else None, 680 brds.get_selected_dict(), args) 681