1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 5# 6 7""" 8Build and query a Kconfig database for boards. 9 10See doc/develop/moveconfig.rst for documentation. 11""" 12 13from argparse import ArgumentParser 14import collections 15from contextlib import ExitStack 16import doctest 17import filecmp 18import fnmatch 19import glob 20import multiprocessing 21import os 22import queue 23import re 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import unittest 31 32import asteval 33from buildman import bsettings 34from buildman import kconfiglib 35from buildman import toolchain 36from u_boot_pylib import terminal 37 38SHOW_GNU_MAKE = 'scripts/show-gnu-make' 39SLEEP_TIME=0.03 40 41STATE_IDLE = 0 42STATE_DEFCONFIG = 1 43STATE_AUTOCONF = 2 44STATE_SAVEDEFCONFIG = 3 45 46AUTO_CONF_PATH = 'include/config/auto.conf' 47CONFIG_DATABASE = 'qconfig.db' 48FAILED_LIST = 'qconfig.failed' 49 50CONFIG_LEN = len('CONFIG_') 51 52SIZES = { 53 'SZ_1': 0x00000001, 'SZ_2': 0x00000002, 54 'SZ_4': 0x00000004, 'SZ_8': 0x00000008, 55 'SZ_16': 0x00000010, 'SZ_32': 0x00000020, 56 'SZ_64': 0x00000040, 'SZ_128': 0x00000080, 57 'SZ_256': 0x00000100, 'SZ_512': 0x00000200, 58 'SZ_1K': 0x00000400, 'SZ_2K': 0x00000800, 59 'SZ_4K': 0x00001000, 'SZ_8K': 0x00002000, 60 'SZ_16K': 0x00004000, 'SZ_32K': 0x00008000, 61 'SZ_64K': 0x00010000, 'SZ_128K': 0x00020000, 62 'SZ_256K': 0x00040000, 'SZ_512K': 0x00080000, 63 'SZ_1M': 0x00100000, 'SZ_2M': 0x00200000, 64 'SZ_4M': 0x00400000, 'SZ_8M': 0x00800000, 65 'SZ_16M': 0x01000000, 'SZ_32M': 0x02000000, 66 'SZ_64M': 0x04000000, 'SZ_128M': 0x08000000, 67 'SZ_256M': 0x10000000, 'SZ_512M': 0x20000000, 68 'SZ_1G': 0x40000000, 'SZ_2G': 0x80000000, 69 'SZ_4G': 0x100000000 70} 71 72RE_REMOVE_DEFCONFIG = re.compile(r'(.*)_defconfig') 73 74# CONFIG symbols present in the build system (from Linux) but not actually used 75# in U-Boot; KCONFIG symbols 76IGNORE_SYMS = ['DEBUG_SECTION_MISMATCH', 'FTRACE_MCOUNT_RECORD', 'GCOV_KERNEL', 77 'GCOV_PROFILE_ALL', 'KALLSYMS', 'KASAN', 'MODVERSIONS', 'SHELL', 78 'TPL_BUILD', 'VPL_BUILD', 'IS_ENABLED', 'FOO', 'IF_ENABLED_INT', 79 'IS_ENABLED_', 'IS_ENABLED_1', 'IS_ENABLED_2', 'IS_ENABLED_3', 80 'SPL_', 'TPL_', 'SPL_FOO', 'TPL_FOO', 'TOOLS_FOO', 81 'ACME', 'SPL_ACME', 'TPL_ACME', 'TRACE_BRANCH_PROFILING', 82 'VAL', '_UNDEFINED', 'SPL_BUILD', ] 83 84SPL_PREFIXES = ['SPL_', 'TPL_', 'VPL_', 'TOOLS_'] 85 86### helper functions ### 87def check_top_directory(): 88 """Exit if we are not at the top of source directory.""" 89 for fname in 'README', 'Licenses': 90 if not os.path.exists(fname): 91 sys.exit('Please run at the top of source directory.') 92 93def check_clean_directory(): 94 """Exit if the source tree is not clean.""" 95 for fname in '.config', 'include/config': 96 if os.path.exists(fname): 97 sys.exit("source tree is not clean, please run 'make mrproper'") 98 99def get_make_cmd(): 100 """Get the command name of GNU Make. 101 102 U-Boot needs GNU Make for building, but the command name is not 103 necessarily "make". (for example, "gmake" on FreeBSD). 104 Returns the most appropriate command name on your system. 105 """ 106 with subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) as proc: 107 ret = proc.communicate() 108 if proc.returncode: 109 sys.exit('GNU Make not found') 110 return ret[0].rstrip() 111 112def get_matched_defconfig(line): 113 """Get the defconfig files that match a pattern 114 115 Args: 116 line (str): Path or filename to match, e.g. 'configs/snow_defconfig' or 117 'k2*_defconfig'. If no directory is provided, 'configs/' is 118 prepended 119 120 Returns: 121 list of str: a list of matching defconfig files 122 """ 123 dirname = os.path.dirname(line) 124 if dirname: 125 pattern = line 126 else: 127 pattern = os.path.join('configs', line) 128 return glob.glob(pattern) + glob.glob(pattern + '_defconfig') 129 130def get_matched_defconfigs(defconfigs_file): 131 """Get all the defconfig files that match the patterns in a file. 132 133 Args: 134 defconfigs_file (str): File containing a list of defconfigs to process, 135 or '-' to read the list from stdin 136 137 Returns: 138 list of str: A list of paths to defconfig files, with no duplicates 139 """ 140 defconfigs = [] 141 with ExitStack() as stack: 142 if defconfigs_file == '-': 143 inf = sys.stdin 144 defconfigs_file = 'stdin' 145 else: 146 inf = stack.enter_context(open(defconfigs_file, encoding='utf-8')) 147 for i, line in enumerate(inf): 148 line = line.strip() 149 if not line: 150 continue # skip blank lines silently 151 if ' ' in line: 152 line = line.split(' ')[0] # handle 'git log' input 153 matched = get_matched_defconfig(line) 154 if not matched: 155 print(f"warning: {defconfigs_file}:{i + 1}: no defconfig matched '{line}'", 156 file=sys.stderr) 157 158 defconfigs += matched 159 160 # use set() to drop multiple matching 161 return [defconfig[len('configs') + 1:] for defconfig in set(defconfigs)] 162 163def get_all_defconfigs(): 164 """Get all the defconfig files under the configs/ directory. 165 166 Returns: 167 list of str: List of paths to defconfig files 168 """ 169 defconfigs = [] 170 for (dirpath, _, filenames) in os.walk('configs'): 171 dirpath = dirpath[len('configs') + 1:] 172 for filename in fnmatch.filter(filenames, '*_defconfig'): 173 defconfigs.append(os.path.join(dirpath, filename)) 174 175 return defconfigs 176 177def write_file(fname, data): 178 """Write data to a file 179 180 Args: 181 fname (str): Filename to write to 182 data (list of str): Lines to write (with or without trailing newline); 183 or str to write 184 """ 185 with open(fname, 'w', encoding='utf-8') as out: 186 if isinstance(data, list): 187 for line in data: 188 print(line.rstrip('\n'), file=out) 189 else: 190 out.write(data) 191 192def read_file(fname, as_lines=True, skip_unicode=False): 193 """Read a file and return the contents 194 195 Args: 196 fname (str): Filename to read from 197 as_lines (bool): Return file contents as a list of lines 198 skip_unicode (bool): True to report unicode errors and continue 199 200 Returns: 201 iter of str: List of ;ines from the file with newline removed; str if 202 as_lines is False with newlines intact; or None if a unicode error 203 occurred 204 205 Raises: 206 UnicodeDecodeError: Unicode error occurred when reading 207 """ 208 with open(fname, encoding='utf-8') as inf: 209 try: 210 if as_lines: 211 return [line.rstrip('\n') for line in inf.readlines()] 212 return inf.read() 213 except UnicodeDecodeError as exc: 214 if not skip_unicode: 215 raise 216 print(f"Failed on file '{fname}: {exc}") 217 return None 218 219def try_expand(line): 220 """If value looks like an expression, try expanding it 221 Otherwise just return the existing value 222 """ 223 if line.find('=') == -1: 224 return line 225 226 try: 227 aeval = asteval.Interpreter( usersyms=SIZES, minimal=True ) 228 cfg, val = re.split("=", line) 229 val= val.strip('\"') 230 if re.search(r'[*+-/]|<<|SZ_+|\(([^\)]+)\)', val): 231 newval = hex(aeval(val)) 232 print(f'\tExpanded expression {val} to {newval}') 233 return cfg+'='+newval 234 except: 235 print(f'\tFailed to expand expression in {line}') 236 237 return line 238 239 240### classes ### 241class Progress: 242 243 """Progress Indicator""" 244 245 def __init__(self, col, total): 246 """Create a new progress indicator. 247 248 Args: 249 color_enabled (bool): True for colour output 250 total (int): A number of defconfig files to process. 251 """ 252 self.col = col 253 self.current = 0 254 self.good = 0 255 self.total = total 256 257 def inc(self, success): 258 """Increment the number of processed defconfig files. 259 260 Args: 261 success (bool): True if processing succeeded 262 """ 263 self.good += success 264 self.current += 1 265 266 def show(self): 267 """Display the progress.""" 268 if self.current != self.total: 269 line = self.col.build(self.col.GREEN, f'{self.good:5d}') 270 line += self.col.build(self.col.RED, 271 f'{self.current - self.good:5d}') 272 line += self.col.build(self.col.MAGENTA, 273 f'/{self.total - self.current}') 274 print(f'{line} \r', end='') 275 sys.stdout.flush() 276 277 278class KconfigScanner: 279 """Kconfig scanner.""" 280 281 def __init__(self): 282 """Scan all the Kconfig files and create a Config object.""" 283 # Define environment variables referenced from Kconfig 284 os.environ['srctree'] = os.getcwd() 285 os.environ['UBOOTVERSION'] = 'dummy' 286 os.environ['KCONFIG_OBJDIR'] = '' 287 os.environ['CC'] = 'gcc' 288 self.conf = kconfiglib.Kconfig() 289 290 291class KconfigParser: 292 293 """A parser of .config and include/autoconf.mk.""" 294 295 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 296 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 297 298 def __init__(self, args, build_dir): 299 """Create a new parser. 300 301 Args: 302 args (Namespace): program arguments 303 build_dir: Build directory. 304 """ 305 self.args = args 306 self.dotconfig = os.path.join(build_dir, '.config') 307 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 308 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', 309 'autoconf.mk') 310 self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH) 311 self.defconfig = os.path.join(build_dir, 'defconfig') 312 313 def get_arch(self): 314 """Parse .config file and return the architecture. 315 316 Returns: 317 Architecture name (e.g. 'arm'). 318 """ 319 arch = '' 320 cpu = '' 321 for line in read_file(self.dotconfig): 322 m_arch = self.re_arch.match(line) 323 if m_arch: 324 arch = m_arch.group(1) 325 continue 326 m_cpu = self.re_cpu.match(line) 327 if m_cpu: 328 cpu = m_cpu.group(1) 329 330 if not arch: 331 return None 332 333 # fix-up for aarch64 334 if arch == 'arm' and cpu == 'armv8': 335 arch = 'aarch64' 336 337 return arch 338 339 340class DatabaseThread(threading.Thread): 341 """This thread processes results from Slot threads. 342 343 It collects the data in the master config directary. There is only one 344 result thread, and this helps to serialise the build output. 345 """ 346 def __init__(self, config_db, db_queue): 347 """Set up a new result thread 348 349 Args: 350 builder: Builder which will be sent each result 351 """ 352 threading.Thread.__init__(self) 353 self.config_db = config_db 354 self.db_queue= db_queue 355 356 def run(self): 357 """Called to start up the result thread. 358 359 We collect the next result job and pass it on to the build. 360 """ 361 while True: 362 defconfig, configs = self.db_queue.get() 363 self.config_db[defconfig] = configs 364 self.db_queue.task_done() 365 366 367class Slot: 368 369 """A slot to store a subprocess. 370 371 Each instance of this class handles one subprocess. 372 This class is useful to control multiple threads 373 for faster processing. 374 """ 375 376 def __init__(self, toolchains, args, progress, devnull, make_cmd, 377 reference_src_dir, db_queue, col): 378 """Create a new process slot. 379 380 Args: 381 toolchains: Toolchains object containing toolchains. 382 args: Program arguments 383 progress: A progress indicator. 384 devnull: A file object of '/dev/null'. 385 make_cmd: command name of GNU Make. 386 reference_src_dir: Determine the true starting config state from this 387 source tree. 388 db_queue: output queue to write config info for the database 389 col (terminal.Color): Colour object 390 """ 391 self.toolchains = toolchains 392 self.args = args 393 self.progress = progress 394 self.build_dir = tempfile.mkdtemp() 395 self.devnull = devnull 396 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 397 self.reference_src_dir = reference_src_dir 398 self.db_queue = db_queue 399 self.col = col 400 self.parser = KconfigParser(args, self.build_dir) 401 self.state = STATE_IDLE 402 self.failed_boards = set() 403 self.defconfig = None 404 self.log = [] 405 self.current_src_dir = None 406 self.proc = None 407 408 def __del__(self): 409 """Delete the working directory 410 411 This function makes sure the temporary directory is cleaned away 412 even if Python suddenly dies due to error. It should be done in here 413 because it is guaranteed the destructor is always invoked when the 414 instance of the class gets unreferenced. 415 416 If the subprocess is still running, wait until it finishes. 417 """ 418 if self.state != STATE_IDLE: 419 while self.proc.poll() is None: 420 pass 421 shutil.rmtree(self.build_dir) 422 423 def add(self, defconfig): 424 """Assign a new subprocess for defconfig and add it to the slot. 425 426 If the slot is vacant, create a new subprocess for processing the 427 given defconfig and add it to the slot. Just returns False if 428 the slot is occupied (i.e. the current subprocess is still running). 429 430 Args: 431 defconfig (str): defconfig name. 432 433 Returns: 434 Return True on success or False on failure 435 """ 436 if self.state != STATE_IDLE: 437 return False 438 439 self.defconfig = defconfig 440 self.log = [] 441 self.current_src_dir = self.reference_src_dir 442 self.do_defconfig() 443 return True 444 445 def poll(self): 446 """Check the status of the subprocess and handle it as needed. 447 448 Returns True if the slot is vacant (i.e. in idle state). 449 If the configuration is successfully finished, assign a new 450 subprocess to build include/autoconf.mk. 451 If include/autoconf.mk is generated, invoke the parser to 452 parse the .config and the include/autoconf.mk, moving 453 config options to the .config as needed. 454 If the .config was updated, run "make savedefconfig" to sync 455 it, update the original defconfig, and then set the slot back 456 to the idle state. 457 458 Returns: 459 Return True if the subprocess is terminated, False otherwise 460 """ 461 if self.state == STATE_IDLE: 462 return True 463 464 if self.proc.poll() is None: 465 return False 466 467 if self.proc.poll() != 0: 468 self.handle_error() 469 elif self.state == STATE_DEFCONFIG: 470 if self.reference_src_dir and not self.current_src_dir: 471 self.do_savedefconfig() 472 else: 473 self.do_autoconf() 474 elif self.state == STATE_AUTOCONF: 475 if self.current_src_dir: 476 self.current_src_dir = None 477 self.do_defconfig() 478 elif self.args.build_db: 479 self.do_build_db() 480 else: 481 self.do_savedefconfig() 482 elif self.state == STATE_SAVEDEFCONFIG: 483 self.update_defconfig() 484 else: 485 sys.exit('Internal Error. This should not happen.') 486 487 return self.state == STATE_IDLE 488 489 def handle_error(self): 490 """Handle error cases.""" 491 492 self.log.append(self.col.build(self.col.RED, 'Failed to process', 493 bright=True)) 494 if self.args.verbose: 495 for line in self.proc.stderr.read().decode().splitlines(): 496 self.log.append(self.col.build(self.col.CYAN, line, True)) 497 self.finish(False) 498 499 def do_defconfig(self): 500 """Run 'make <board>_defconfig' to create the .config file.""" 501 502 cmd = list(self.make_cmd) 503 cmd.append(self.defconfig) 504 self.proc = subprocess.Popen(cmd, stdout=self.devnull, 505 stderr=subprocess.PIPE, 506 cwd=self.current_src_dir) 507 self.state = STATE_DEFCONFIG 508 509 def do_autoconf(self): 510 """Run 'make AUTO_CONF_PATH'.""" 511 512 arch = self.parser.get_arch() 513 try: 514 tchain = self.toolchains.Select(arch) 515 except ValueError: 516 self.log.append(self.col.build( 517 self.col.YELLOW, 518 f"Tool chain for '{arch}' is missing: do nothing")) 519 self.finish(False) 520 return 521 env = tchain.MakeEnvironment(False) 522 523 cmd = list(self.make_cmd) 524 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 525 cmd.append(AUTO_CONF_PATH) 526 self.proc = subprocess.Popen(cmd, stdout=self.devnull, env=env, 527 stderr=subprocess.PIPE, 528 cwd=self.current_src_dir) 529 self.state = STATE_AUTOCONF 530 531 def do_build_db(self): 532 """Add the board to the database""" 533 configs = {} 534 for line in read_file(os.path.join(self.build_dir, AUTO_CONF_PATH)): 535 if line.startswith('CONFIG'): 536 config, value = line.split('=', 1) 537 configs[config] = value.rstrip() 538 self.db_queue.put([self.defconfig, configs]) 539 self.finish(True) 540 541 def do_savedefconfig(self): 542 """Update the .config and run 'make savedefconfig'.""" 543 if not self.args.force_sync: 544 self.finish(True) 545 return 546 547 cmd = list(self.make_cmd) 548 cmd.append('savedefconfig') 549 self.proc = subprocess.Popen(cmd, stdout=self.devnull, 550 stderr=subprocess.PIPE) 551 self.state = STATE_SAVEDEFCONFIG 552 553 def update_defconfig(self): 554 """Update the input defconfig and go back to the idle state.""" 555 orig_defconfig = os.path.join('configs', self.defconfig) 556 new_defconfig = os.path.join(self.build_dir, 'defconfig') 557 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 558 559 if updated: 560 self.log.append( 561 self.col.build(self.col.BLUE, 'defconfig updated', bright=True)) 562 563 if not self.args.dry_run and updated: 564 shutil.move(new_defconfig, orig_defconfig) 565 self.finish(True) 566 567 def finish(self, success): 568 """Display log along with progress and go to the idle state. 569 570 Args: 571 success (bool): Should be True when the defconfig was processed 572 successfully, or False when it fails. 573 """ 574 # output at least 30 characters to hide the "* defconfigs out of *". 575 name = self.defconfig[:-len('_defconfig')] 576 if self.log: 577 578 # Put the first log line on the first line 579 log = name.ljust(20) + ' ' + self.log[0] 580 581 if len(self.log) > 1: 582 log += '\n' + '\n'.join([' ' + s for s in self.log[1:]]) 583 # Some threads are running in parallel. 584 # Print log atomically to not mix up logs from different threads. 585 print(log, file=(sys.stdout if success else sys.stderr)) 586 587 if not success: 588 if self.args.exit_on_error: 589 sys.exit('Exit on error.') 590 # If --exit-on-error flag is not set, skip this board and continue. 591 # Record the failed board. 592 self.failed_boards.add(name) 593 594 self.progress.inc(success) 595 self.progress.show() 596 self.state = STATE_IDLE 597 598 def get_failed_boards(self): 599 """Returns a set of failed boards (defconfigs) in this slot. 600 """ 601 return self.failed_boards 602 603class Slots: 604 605 """Controller of the array of subprocess slots.""" 606 607 def __init__(self, toolchains, args, progress, reference_src_dir, db_queue, 608 col): 609 """Create a new slots controller. 610 611 Args: 612 toolchains (Toolchains): Toolchains object containing toolchains 613 args (Namespace): Program arguments 614 progress (Progress): A progress indicator. 615 reference_src_dir (str): Determine the true starting config state 616 from this source tree (None for none) 617 db_queue (Queue): output queue to write config info for the database 618 col (terminal.Color): Colour object 619 """ 620 self.args = args 621 self.slots = [] 622 self.progress = progress 623 self.col = col 624 devnull = subprocess.DEVNULL 625 make_cmd = get_make_cmd() 626 for _ in range(args.jobs): 627 self.slots.append(Slot(toolchains, args, progress, devnull, 628 make_cmd, reference_src_dir, db_queue, col)) 629 630 def add(self, defconfig): 631 """Add a new subprocess if a vacant slot is found. 632 633 Args: 634 defconfig (str): defconfig name to be put into. 635 636 Returns: 637 Return True on success or False on failure 638 """ 639 for slot in self.slots: 640 if slot.add(defconfig): 641 return True 642 return False 643 644 def available(self): 645 """Check if there is a vacant slot. 646 647 Returns: 648 Return True if at lease one vacant slot is found, False otherwise. 649 """ 650 for slot in self.slots: 651 if slot.poll(): 652 return True 653 return False 654 655 def empty(self): 656 """Check if all slots are vacant. 657 658 Returns: 659 Return True if all the slots are vacant, False otherwise. 660 """ 661 ret = True 662 for slot in self.slots: 663 if not slot.poll(): 664 ret = False 665 return ret 666 667 def write_failed_boards(self): 668 """Show the results of processing""" 669 boards = set() 670 671 for slot in self.slots: 672 boards |= slot.get_failed_boards() 673 674 if boards: 675 boards = '\n'.join(sorted(boards)) + '\n' 676 write_file(FAILED_LIST, boards) 677 678 679class ReferenceSource: 680 681 """Reference source against which original configs should be parsed.""" 682 683 def __init__(self, commit): 684 """Create a reference source directory based on a specified commit. 685 686 Args: 687 commit: commit to git-clone 688 """ 689 self.src_dir = tempfile.mkdtemp() 690 print('Cloning git repo to a separate work directory...') 691 subprocess.check_output(['git', 'clone', os.getcwd(), '.'], 692 cwd=self.src_dir) 693 rev = subprocess.check_output(['git', 'rev-parse', '--short', 694 commit]).strip() 695 print(f"Checkout '{rev}' to build the original autoconf.mk.") 696 subprocess.check_output(['git', 'checkout', commit], 697 stderr=subprocess.STDOUT, cwd=self.src_dir) 698 699 def __del__(self): 700 """Delete the reference source directory 701 702 This function makes sure the temporary directory is cleaned away 703 even if Python suddenly dies due to error. It should be done in here 704 because it is guaranteed the destructor is always invoked when the 705 instance of the class gets unreferenced. 706 """ 707 shutil.rmtree(self.src_dir) 708 709 def get_dir(self): 710 """Return the absolute path to the reference source directory.""" 711 712 return self.src_dir 713 714def move_config(toolchains, args, db_queue, col): 715 """Build database or sync config options to defconfig files. 716 717 Args: 718 toolchains (Toolchains): Toolchains to use 719 args (Namespace): Program arguments 720 db_queue (Queue): Queue for database updates 721 col (terminal.Color): Colour object 722 723 Returns: 724 Progress: Progress indicator 725 """ 726 if args.git_ref: 727 reference_src = ReferenceSource(args.git_ref) 728 reference_src_dir = reference_src.get_dir() 729 else: 730 reference_src_dir = None 731 732 if args.defconfigs: 733 defconfigs = get_matched_defconfigs(args.defconfigs) 734 else: 735 defconfigs = get_all_defconfigs() 736 737 progress = Progress(col, len(defconfigs)) 738 slots = Slots(toolchains, args, progress, reference_src_dir, db_queue, col) 739 740 # Main loop to process defconfig files: 741 # Add a new subprocess into a vacant slot. 742 # Sleep if there is no available slot. 743 for defconfig in defconfigs: 744 while not slots.add(defconfig): 745 while not slots.available(): 746 # No available slot: sleep for a while 747 time.sleep(SLEEP_TIME) 748 749 # wait until all the subprocesses finish 750 while not slots.empty(): 751 time.sleep(SLEEP_TIME) 752 753 slots.write_failed_boards() 754 return progress 755 756def find_kconfig_rules(kconf, config, imply_config): 757 """Check whether a config has a 'select' or 'imply' keyword 758 759 Args: 760 kconf (Kconfiglib.Kconfig): Kconfig object 761 config (str): Name of config to check (without CONFIG_ prefix) 762 imply_config (str): Implying config (without CONFIG_ prefix) which may 763 or may not have an 'imply' for 'config') 764 765 Returns: 766 Symbol object for 'config' if found, else None 767 """ 768 sym = kconf.syms.get(imply_config) 769 if sym: 770 for sel, _ in (sym.selects + sym.implies): 771 if sel.name == config: 772 return sym 773 return None 774 775def check_imply_rule(kconf, config, imply_config): 776 """Check if we can add an 'imply' option 777 778 This finds imply_config in the Kconfig and looks to see if it is possible 779 to add an 'imply' for 'config' to that part of the Kconfig. 780 781 Args: 782 kconf (Kconfiglib.Kconfig): Kconfig object 783 config (str): Name of config to check (without CONFIG_ prefix) 784 imply_config (str): Implying config (without CONFIG_ prefix) which may 785 or may not have an 'imply' for 'config') 786 787 Returns: 788 tuple: 789 str: filename of Kconfig file containing imply_config, or None if 790 none 791 int: line number within the Kconfig file, or 0 if none 792 str: message indicating the result 793 """ 794 sym = kconf.syms.get(imply_config) 795 if not sym: 796 return 'cannot find sym' 797 nodes = sym.nodes 798 if len(nodes) != 1: 799 return f'{len(nodes)} locations' 800 node = nodes[0] 801 fname, linenum = node.filename, node.linenr 802 cwd = os.getcwd() 803 if cwd and fname.startswith(cwd): 804 fname = fname[len(cwd) + 1:] 805 file_line = f' at {fname}:{linenum}' 806 data = read_file(fname) 807 if data[linenum - 1] != f'config {imply_config}': 808 return None, 0, f'bad sym format {data[linenum]}{file_line})' 809 return fname, linenum, f'adding{file_line}' 810 811def add_imply_rule(config, fname, linenum): 812 """Add a new 'imply' option to a Kconfig 813 814 Args: 815 config (str): config option to add an imply for (without CONFIG_ prefix) 816 fname (str): Kconfig filename to update 817 linenum (int): Line number to place the 'imply' before 818 819 Returns: 820 Message indicating the result 821 """ 822 file_line = f' at {fname}:{linenum}' 823 data = read_file(fname) 824 linenum -= 1 825 826 for offset, line in enumerate(data[linenum:]): 827 if line.strip().startswith('help') or not line: 828 data.insert(linenum + offset, f'\timply {config}') 829 write_file(fname, data) 830 return f'added{file_line}' 831 832 return 'could not insert%s' 833 834(IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = ( 835 1, 2, 4, 8) 836 837IMPLY_FLAGS = { 838 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'], 839 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'], 840 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'], 841 'non-arch-board': [ 842 IMPLY_NON_ARCH_BOARD, 843 'Allow Kconfig options outside arch/ and /board/ to imply'], 844} 845 846 847def read_database(): 848 """Read in the config database 849 850 Returns: 851 tuple: 852 set of all config options seen (each a str) 853 set of all defconfigs seen (each a str) 854 dict of configs for each defconfig: 855 key: defconfig name, e.g. "MPC8548CDS_legacy_defconfig" 856 value: dict: 857 key: CONFIG option 858 value: Value of option 859 dict of defconfigs for each config: 860 key: CONFIG option 861 value: set of boards using that option 862 863 """ 864 configs = {} 865 866 # key is defconfig name, value is dict of (CONFIG_xxx, value) 867 config_db = {} 868 869 # Set of all config options we have seen 870 all_configs = set() 871 872 # Set of all defconfigs we have seen 873 all_defconfigs = set() 874 875 defconfig_db = collections.defaultdict(set) 876 for line in read_file(CONFIG_DATABASE): 877 line = line.rstrip() 878 if not line: # Separator between defconfigs 879 config_db[defconfig] = configs 880 all_defconfigs.add(defconfig) 881 configs = {} 882 elif line[0] == ' ': # CONFIG line 883 config, value = line.strip().split('=', 1) 884 configs[config] = value 885 defconfig_db[config].add(defconfig) 886 all_configs.add(config) 887 else: # New defconfig 888 defconfig = line 889 890 return all_configs, all_defconfigs, config_db, defconfig_db 891 892 893def do_imply_config(config_list, add_imply, imply_flags, skip_added, 894 check_kconfig=True, find_superset=False): 895 """Find CONFIG options which imply those in the list 896 897 Some CONFIG options can be implied by others and this can help to reduce 898 the size of the defconfig files. For example, CONFIG_X86 implies 899 CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and 900 all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to 901 each of the x86 defconfig files. 902 903 This function uses the qconfig database to find such options. It 904 displays a list of things that could possibly imply those in the list. 905 The algorithm ignores any that start with CONFIG_TARGET since these 906 typically refer to only a few defconfigs (often one). It also does not 907 display a config with less than 5 defconfigs. 908 909 The algorithm works using sets. For each target config in config_list: 910 - Get the set 'defconfigs' which use that target config 911 - For each config (from a list of all configs): 912 - Get the set 'imply_defconfig' of defconfigs which use that config 913 - 914 - If imply_defconfigs contains anything not in defconfigs then 915 this config does not imply the target config 916 917 Params: 918 config_list: List of CONFIG options to check (each a string) 919 add_imply: Automatically add an 'imply' for each config. 920 imply_flags: Flags which control which implying configs are allowed 921 (IMPLY_...) 922 skip_added: Don't show options which already have an imply added. 923 check_kconfig: Check if implied symbols already have an 'imply' or 924 'select' for the target config, and show this information if so. 925 find_superset: True to look for configs which are a superset of those 926 already found. So for example if CONFIG_EXYNOS5 implies an option, 927 but CONFIG_EXYNOS covers a larger set of defconfigs and also 928 implies that option, this will drop the former in favour of the 929 latter. In practice this option has not proved very used. 930 931 Note the terminoloy: 932 config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM') 933 defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig') 934 """ 935 kconf = KconfigScanner().conf if check_kconfig else None 936 if add_imply and add_imply != 'all': 937 add_imply = add_imply.split(',') 938 939 all_configs, all_defconfigs, _, defconfig_db = read_database() 940 941 # Work through each target config option in turn, independently 942 for config in config_list: 943 defconfigs = defconfig_db.get(config) 944 if not defconfigs: 945 print(f'{config} not found in any defconfig') 946 continue 947 948 # Get the set of defconfigs without this one (since a config cannot 949 # imply itself) 950 non_defconfigs = all_defconfigs - defconfigs 951 num_defconfigs = len(defconfigs) 952 print(f'{config} found in {num_defconfigs}/{len(all_configs)} defconfigs') 953 954 # This will hold the results: key=config, value=defconfigs containing it 955 imply_configs = {} 956 rest_configs = all_configs - set([config]) 957 958 # Look at every possible config, except the target one 959 for imply_config in rest_configs: 960 if 'ERRATUM' in imply_config: 961 continue 962 if not imply_flags & IMPLY_CMD: 963 if 'CONFIG_CMD' in imply_config: 964 continue 965 if not imply_flags & IMPLY_TARGET: 966 if 'CONFIG_TARGET' in imply_config: 967 continue 968 969 # Find set of defconfigs that have this config 970 imply_defconfig = defconfig_db[imply_config] 971 972 # Get the intersection of this with defconfigs containing the 973 # target config 974 common_defconfigs = imply_defconfig & defconfigs 975 976 # Get the set of defconfigs containing this config which DO NOT 977 # also contain the taret config. If this set is non-empty it means 978 # that this config affects other defconfigs as well as (possibly) 979 # the ones affected by the target config. This means it implies 980 # things we don't want to imply. 981 not_common_defconfigs = imply_defconfig & non_defconfigs 982 if not_common_defconfigs: 983 continue 984 985 # If there are common defconfigs, imply_config may be useful 986 if common_defconfigs: 987 skip = False 988 if find_superset: 989 for prev in list(imply_configs.keys()): 990 prev_count = len(imply_configs[prev]) 991 count = len(common_defconfigs) 992 if (prev_count > count and 993 (imply_configs[prev] & common_defconfigs == 994 common_defconfigs)): 995 # skip imply_config because prev is a superset 996 skip = True 997 break 998 if count > prev_count: 999 # delete prev because imply_config is a superset 1000 del imply_configs[prev] 1001 if not skip: 1002 imply_configs[imply_config] = common_defconfigs 1003 1004 # Now we have a dict imply_configs of configs which imply each config 1005 # The value of each dict item is the set of defconfigs containing that 1006 # config. Rank them so that we print the configs that imply the largest 1007 # number of defconfigs first. 1008 ranked_iconfigs = sorted(imply_configs, 1009 key=lambda k: len(imply_configs[k]), reverse=True) 1010 kconfig_info = '' 1011 cwd = os.getcwd() 1012 add_list = collections.defaultdict(list) 1013 for iconfig in ranked_iconfigs: 1014 num_common = len(imply_configs[iconfig]) 1015 1016 # Don't bother if there are less than 5 defconfigs affected. 1017 if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5): 1018 continue 1019 missing = defconfigs - imply_configs[iconfig] 1020 missing_str = ', '.join(missing) if missing else 'all' 1021 missing_str = '' 1022 show = True 1023 if kconf: 1024 sym = find_kconfig_rules(kconf, config[CONFIG_LEN:], 1025 iconfig[CONFIG_LEN:]) 1026 kconfig_info = '' 1027 if sym: 1028 nodes = sym.nodes 1029 if len(nodes) == 1: 1030 fname, linenum = nodes[0].filename, nodes[0].linenr 1031 if cwd and fname.startswith(cwd): 1032 fname = fname[len(cwd) + 1:] 1033 kconfig_info = f'{fname}:{linenum}' 1034 if skip_added: 1035 show = False 1036 else: 1037 sym = kconf.syms.get(iconfig[CONFIG_LEN:]) 1038 fname = '' 1039 if sym: 1040 nodes = sym.nodes 1041 if len(nodes) == 1: 1042 fname, linenum = nodes[0].filename, nodes[0].linenr 1043 if cwd and fname.startswith(cwd): 1044 fname = fname[len(cwd) + 1:] 1045 in_arch_board = not sym or (fname.startswith('arch') or 1046 fname.startswith('board')) 1047 if (not in_arch_board and 1048 not imply_flags & IMPLY_NON_ARCH_BOARD): 1049 continue 1050 1051 if add_imply and (add_imply == 'all' or 1052 iconfig in add_imply): 1053 fname, linenum, kconfig_info = (check_imply_rule(kconf, 1054 config[CONFIG_LEN:], iconfig[CONFIG_LEN:])) 1055 if fname: 1056 add_list[fname].append(linenum) 1057 1058 if show and kconfig_info != 'skip': 1059 print(f'{num_common:5d} : ' 1060 f'{iconfig.ljust(30):-30s}{kconfig_info:-25s} {missing_str}') 1061 1062 # Having collected a list of things to add, now we add them. We process 1063 # each file from the largest line number to the smallest so that 1064 # earlier additions do not affect our line numbers. E.g. if we added an 1065 # imply at line 20 it would change the position of each line after 1066 # that. 1067 for fname, linenums in add_list.items(): 1068 for linenum in sorted(linenums, reverse=True): 1069 add_imply_rule(config[CONFIG_LEN:], fname, linenum) 1070 1071def defconfig_matches(configs, re_match): 1072 """Check if any CONFIG option matches a regex 1073 1074 The match must be complete, i.e. from the start to end of the CONFIG option. 1075 1076 Args: 1077 configs (dict): Dict of CONFIG options: 1078 key: CONFIG option 1079 value: Value of option 1080 re_match (re.Pattern): Match to check 1081 1082 Returns: 1083 bool: True if any CONFIG matches the regex 1084 """ 1085 for cfg in configs: 1086 if re_match.fullmatch(cfg): 1087 return True 1088 return False 1089 1090def do_find_config(config_list): 1091 """Find boards with a given combination of CONFIGs 1092 1093 Params: 1094 config_list: List of CONFIG options to check (each a regex consisting 1095 of a config option, with or without a CONFIG_ prefix. If an option 1096 is preceded by a tilde (~) then it must be false, otherwise it must 1097 be true) 1098 """ 1099 _, all_defconfigs, config_db, _ = read_database() 1100 1101 # Start with all defconfigs 1102 out = all_defconfigs 1103 1104 # Work through each config in turn 1105 for item in config_list: 1106 # Get the real config name and whether we want this config or not 1107 cfg = item 1108 want = True 1109 if cfg[0] == '~': 1110 want = False 1111 cfg = cfg[1:] 1112 1113 # Search everything that is still in the running. If it has a config 1114 # that we want, or doesn't have one that we don't, add it into the 1115 # running for the next stage 1116 in_list = out 1117 out = set() 1118 re_match = re.compile(cfg) 1119 for defc in in_list: 1120 has_cfg = defconfig_matches(config_db[defc], re_match) 1121 if has_cfg == want: 1122 out.add(defc) 1123 print(f'{len(out)} matches') 1124 print(' '.join(item.split('_defconfig')[0] for item in out)) 1125 1126 1127def prefix_config(cfg): 1128 """Prefix a config with CONFIG_ if needed 1129 1130 This handles ~ operator, which indicates that the CONFIG should be disabled 1131 1132 >>> prefix_config('FRED') 1133 'CONFIG_FRED' 1134 >>> prefix_config('CONFIG_FRED') 1135 'CONFIG_FRED' 1136 >>> prefix_config('~FRED') 1137 '~CONFIG_FRED' 1138 >>> prefix_config('~CONFIG_FRED') 1139 '~CONFIG_FRED' 1140 >>> prefix_config('A123') 1141 'CONFIG_A123' 1142 """ 1143 oper = '' 1144 if cfg[0] == '~': 1145 oper = cfg[0] 1146 cfg = cfg[1:] 1147 if not cfg.startswith('CONFIG_'): 1148 cfg = 'CONFIG_' + cfg 1149 return oper + cfg 1150 1151 1152RE_MK_CONFIGS = re.compile(r'CONFIG_(\$\(SPL_(?:TPL_)?\))?([A-Za-z0-9_]*)') 1153RE_IFDEF = re.compile(r'(ifdef|ifndef)') 1154RE_C_CONFIGS = re.compile(r'CONFIG_([A-Za-z0-9_]*)') 1155RE_CONFIG_IS = re.compile(r'CONFIG_IS_ENABLED\(([A-Za-z0-9_]*)\)') 1156 1157class ConfigUse: 1158 def __init__(self, cfg, is_spl, fname, rest): 1159 self.cfg = cfg 1160 self.is_spl = is_spl 1161 self.fname = fname 1162 self.rest = rest 1163 1164 def __hash__(self): 1165 return hash((self.cfg, self.is_spl)) 1166 1167def scan_makefiles(fnames): 1168 """Scan Makefiles looking for Kconfig options 1169 1170 Looks for uses of CONFIG options in Makefiles 1171 1172 Args: 1173 fnames (list of tuple): 1174 str: Makefile filename where the option was found 1175 str: Line of the Makefile 1176 1177 Returns: 1178 tuple: 1179 dict: all_uses 1180 key (ConfigUse): object 1181 value (list of str): matching lines 1182 dict: Uses by filename 1183 key (str): filename 1184 value (set of ConfigUse): uses in that filename 1185 1186 >>> RE_MK_CONFIGS.search('CONFIG_FRED').groups() 1187 (None, 'FRED') 1188 >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_)MARY').groups() 1189 ('$(SPL_)', 'MARY') 1190 >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_TPL_)MARY').groups() 1191 ('$(SPL_TPL_)', 'MARY') 1192 """ 1193 all_uses = collections.defaultdict(list) 1194 fname_uses = {} 1195 for fname, rest in fnames: 1196 m_iter = RE_MK_CONFIGS.finditer(rest) 1197 for mat in m_iter: 1198 real_opt = mat.group(2) 1199 if real_opt == '': 1200 continue 1201 is_spl = False 1202 if mat.group(1): 1203 is_spl = True 1204 use = ConfigUse(real_opt, is_spl, fname, rest) 1205 if fname not in fname_uses: 1206 fname_uses[fname] = set() 1207 fname_uses[fname].add(use) 1208 all_uses[use].append(rest) 1209 return all_uses, fname_uses 1210 1211 1212def scan_src_files(fnames): 1213 """Scan source files (other than Makefiles) looking for Kconfig options 1214 1215 Looks for uses of CONFIG options 1216 1217 Args: 1218 fnames (list of tuple): 1219 str: Makefile filename where the option was found 1220 str: Line of the Makefile 1221 1222 Returns: 1223 tuple: 1224 dict: all_uses 1225 key (ConfigUse): object 1226 value (list of str): matching lines 1227 dict: Uses by filename 1228 key (str): filename 1229 value (set of ConfigUse): uses in that filename 1230 1231 >>> RE_C_CONFIGS.search('CONFIG_FRED').groups() 1232 ('FRED',) 1233 >>> RE_CONFIG_IS.search('CONFIG_IS_ENABLED(MARY)').groups() 1234 ('MARY',) 1235 >>> RE_CONFIG_IS.search('#if CONFIG_IS_ENABLED(OF_PLATDATA)').groups() 1236 ('OF_PLATDATA',) 1237 """ 1238 fname = None 1239 rest = None 1240 1241 def add_uses(m_iter, is_spl): 1242 for mat in m_iter: 1243 real_opt = mat.group(1) 1244 if real_opt == '': 1245 continue 1246 use = ConfigUse(real_opt, is_spl, fname, rest) 1247 if fname not in fname_uses: 1248 fname_uses[fname] = set() 1249 fname_uses[fname].add(use) 1250 all_uses[use].append(rest) 1251 1252 all_uses = collections.defaultdict(list) 1253 fname_uses = {} 1254 for fname, rest in fnames: 1255 m_iter = RE_C_CONFIGS.finditer(rest) 1256 add_uses(m_iter, False) 1257 1258 m_iter2 = RE_CONFIG_IS.finditer(rest) 1259 add_uses(m_iter2, True) 1260 1261 return all_uses, fname_uses 1262 1263 1264MODE_NORMAL, MODE_SPL, MODE_PROPER = range(3) 1265 1266def do_scan_source(path, do_update): 1267 """Scan the source tree for Kconfig inconsistencies 1268 1269 Args: 1270 path (str): Path to source tree 1271 do_update (bool) : True to write to scripts/kconf_... files 1272 """ 1273 def is_not_proper(name): 1274 for prefix in SPL_PREFIXES: 1275 if name.startswith(prefix): 1276 return name[len(prefix):] 1277 return False 1278 1279 def check_not_found(all_uses, spl_mode): 1280 """Check for Kconfig options mentioned in the source but not in Kconfig 1281 1282 Args: 1283 all_uses (dict): 1284 key (ConfigUse): object 1285 value (list of str): matching lines 1286 spl_mode (int): If MODE_SPL, look at source code which implies 1287 an SPL_ option, but for which there is none; 1288 for MOD_PROPER, look at source code which implies a Proper 1289 option (i.e. use of CONFIG_IS_ENABLED() or $(SPL_) or 1290 $(SPL_TPL_) but for which there none; 1291 if MODE_NORMAL, ignore SPL 1292 1293 Returns: 1294 dict: 1295 key (str): CONFIG name (without 'CONFIG_' prefix 1296 value (list of ConfigUse): List of uses of this CONFIG 1297 """ 1298 # Make sure we know about all the options 1299 not_found = collections.defaultdict(list) 1300 for use, _ in all_uses.items(): 1301 name = use.cfg 1302 if name in IGNORE_SYMS: 1303 continue 1304 check = True 1305 1306 if spl_mode == MODE_SPL: 1307 check = use.is_spl 1308 1309 # If it is an SPL symbol, try prepending all SPL_ prefixes to 1310 # find at least one SPL symbol 1311 if use.is_spl: 1312 for prefix in SPL_PREFIXES: 1313 try_name = prefix + name 1314 sym = kconf.syms.get(try_name) 1315 if sym: 1316 break 1317 if not sym: 1318 not_found[f'SPL_{name}'].append(use) 1319 continue 1320 elif spl_mode == MODE_PROPER: 1321 # Try to find the Proper version of this symbol, i.e. without 1322 # the SPL_ prefix 1323 proper_name = is_not_proper(name) 1324 if proper_name: 1325 name = proper_name 1326 elif not use.is_spl: 1327 check = False 1328 else: # MODE_NORMAL 1329 sym = kconf.syms.get(name) 1330 if not sym: 1331 proper_name = is_not_proper(name) 1332 if proper_name: 1333 name = proper_name 1334 sym = kconf.syms.get(name) 1335 if not sym: 1336 for prefix in SPL_PREFIXES: 1337 try_name = prefix + name 1338 sym = kconf.syms.get(try_name) 1339 if sym: 1340 break 1341 if not sym: 1342 not_found[name].append(use) 1343 continue 1344 1345 sym = kconf.syms.get(name) 1346 if not sym and check: 1347 not_found[name].append(use) 1348 return not_found 1349 1350 def show_uses(uses): 1351 """Show a list of uses along with their filename and code snippet 1352 1353 Args: 1354 uses (dict): 1355 key (str): CONFIG name (without 'CONFIG_' prefix 1356 value (list of ConfigUse): List of uses of this CONFIG 1357 """ 1358 for name in sorted(uses): 1359 print(f'{name}: ', end='') 1360 for i, use in enumerate(uses[name]): 1361 print(f'{" " if i else ""}{use.fname}: {use.rest.strip()}') 1362 1363 1364 print('Scanning Kconfig') 1365 kconf = KconfigScanner().conf 1366 print(f'Scanning source in {path}') 1367 args = ['git', 'grep', '-E', r'IS_ENABLED|\bCONFIG'] 1368 with subprocess.Popen(args, stdout=subprocess.PIPE) as proc: 1369 out, _ = proc.communicate() 1370 lines = out.splitlines() 1371 re_fname = re.compile('^([^:]*):(.*)') 1372 src_list = [] 1373 mk_list = [] 1374 for line in lines: 1375 linestr = line.decode('utf-8') 1376 m_fname = re_fname.search(linestr) 1377 if not m_fname: 1378 continue 1379 fname, rest = m_fname.groups() 1380 dirname, leaf = os.path.split(fname) 1381 root, ext = os.path.splitext(leaf) 1382 if ext == '.autoconf': 1383 pass 1384 elif ext in ['.c', '.h', '.S', '.lds', '.dts', '.dtsi', '.asl', '.cfg', 1385 '.env', '.tmpl']: 1386 src_list.append([fname, rest]) 1387 elif 'Makefile' in root or ext == '.mk': 1388 mk_list.append([fname, rest]) 1389 elif ext in ['.yml', '.sh', '.py', '.awk', '.pl', '.rst', '', '.sed']: 1390 pass 1391 elif 'Kconfig' in root or 'Kbuild' in root: 1392 pass 1393 elif 'README' in root: 1394 pass 1395 elif dirname in ['configs']: 1396 pass 1397 elif dirname.startswith('doc') or dirname.startswith('scripts/kconfig'): 1398 pass 1399 else: 1400 print(f'Not sure how to handle file {fname}') 1401 1402 # Scan the Makefiles 1403 all_uses, _ = scan_makefiles(mk_list) 1404 1405 spl_not_found = set() 1406 proper_not_found = set() 1407 1408 # Make sure we know about all the options 1409 print('\nCONFIG options present in Makefiles but not Kconfig:') 1410 not_found = check_not_found(all_uses, MODE_NORMAL) 1411 show_uses(not_found) 1412 1413 print('\nCONFIG options present in Makefiles but not Kconfig (SPL):') 1414 not_found = check_not_found(all_uses, MODE_SPL) 1415 show_uses(not_found) 1416 spl_not_found |= {is_not_proper(key) or key for key in not_found.keys()} 1417 1418 print('\nCONFIG options used as Proper in Makefiles but without a non-SPL_ variant:') 1419 not_found = check_not_found(all_uses, MODE_PROPER) 1420 show_uses(not_found) 1421 proper_not_found |= {not_found.keys()} 1422 1423 # Scan the source code 1424 all_uses, _ = scan_src_files(src_list) 1425 1426 # Make sure we know about all the options 1427 print('\nCONFIG options present in source but not Kconfig:') 1428 not_found = check_not_found(all_uses, MODE_NORMAL) 1429 show_uses(not_found) 1430 1431 print('\nCONFIG options present in source but not Kconfig (SPL):') 1432 not_found = check_not_found(all_uses, MODE_SPL) 1433 show_uses(not_found) 1434 spl_not_found |= {is_not_proper(key) or key for key in not_found.keys()} 1435 1436 print('\nCONFIG options used as Proper in source but without a non-SPL_ variant:') 1437 not_found = check_not_found(all_uses, MODE_PROPER) 1438 show_uses(not_found) 1439 proper_not_found |= {not_found.keys()} 1440 1441 print('\nCONFIG options used as SPL but without an SPL_ variant:') 1442 for item in sorted(spl_not_found): 1443 print(f' {item}') 1444 1445 print('\nCONFIG options used as Proper but without a non-SPL_ variant:') 1446 for item in sorted(proper_not_found): 1447 print(f' {item}') 1448 1449 # Write out the updated information 1450 if do_update: 1451 with open(os.path.join(path, 'scripts', 'conf_nospl'), 'w', 1452 encoding='utf-8') as out: 1453 print('# These options should not be enabled in SPL builds\n', 1454 file=out) 1455 for item in sorted(spl_not_found): 1456 print(item, file=out) 1457 with open(os.path.join(path, 'scripts', 'conf_noproper'), 'w', 1458 encoding='utf-8') as out: 1459 print('# These options should not be enabled in Proper builds\n', 1460 file=out) 1461 for item in sorted(proper_not_found): 1462 print(item, file=out) 1463 1464 1465def main(): 1466 try: 1467 cpu_count = multiprocessing.cpu_count() 1468 except NotImplementedError: 1469 cpu_count = 1 1470 1471 epilog = '''Move config options from headers to defconfig files. See 1472doc/develop/moveconfig.rst for documentation.''' 1473 1474 parser = ArgumentParser(epilog=epilog) 1475 # Add arguments here 1476 parser.add_argument('-a', '--add-imply', type=str, default='', 1477 help='comma-separated list of CONFIG options to add ' 1478 "an 'imply' statement to for the CONFIG in -i") 1479 parser.add_argument('-A', '--skip-added', action='store_true', default=False, 1480 help="don't show options which are already marked as " 1481 'implying others') 1482 parser.add_argument('-b', '--build-db', action='store_true', default=False, 1483 help='build a CONFIG database') 1484 parser.add_argument('-C', '--commit', action='store_true', default=False, 1485 help='Create a git commit for the operation') 1486 parser.add_argument('--nocolour', action='store_true', default=False, 1487 help="don't display the log in colour") 1488 parser.add_argument('-d', '--defconfigs', type=str, 1489 help='a file containing a list of defconfigs to move, ' 1490 "one per line (for example 'snow_defconfig') " 1491 "or '-' to read from stdin") 1492 parser.add_argument('-e', '--exit-on-error', action='store_true', 1493 default=False, 1494 help='exit immediately on any error') 1495 parser.add_argument('-f', '--find', action='store_true', default=False, 1496 help='Find boards with a given config combination') 1497 parser.add_argument('-i', '--imply', action='store_true', default=False, 1498 help='find options which imply others') 1499 parser.add_argument('-I', '--imply-flags', type=str, default='', 1500 help="control the -i option ('help' for help") 1501 parser.add_argument('-j', '--jobs', type=int, default=cpu_count, 1502 help='the number of jobs to run simultaneously') 1503 parser.add_argument('-n', '--dry-run', action='store_true', default=False, 1504 help='perform a trial run (show log with no changes)') 1505 parser.add_argument('-r', '--git-ref', type=str, 1506 help='the git ref to clone for building the autoconf.mk') 1507 parser.add_argument('-s', '--force-sync', action='store_true', default=False, 1508 help='force sync by savedefconfig') 1509 parser.add_argument('-S', '--spl', action='store_true', default=False, 1510 help='parse config options defined for SPL build') 1511 parser.add_argument('--scan-source', action='store_true', default=False, 1512 help='scan source for uses of CONFIG options') 1513 parser.add_argument('-t', '--test', action='store_true', default=False, 1514 help='run unit tests') 1515 parser.add_argument('-y', '--yes', action='store_true', default=False, 1516 help="respond 'yes' to any prompts") 1517 parser.add_argument('-u', '--update', action='store_true', default=False, 1518 help="update scripts/ files (use with --scan-source)") 1519 parser.add_argument('-v', '--verbose', action='store_true', default=False, 1520 help='show any build errors as boards are built') 1521 parser.add_argument('configs', nargs='*') 1522 1523 args = parser.parse_args() 1524 1525 if args.test: 1526 sys.argv = [sys.argv[0]] 1527 fail, _ = doctest.testmod() 1528 if fail: 1529 return 1 1530 unittest.main() 1531 1532 col = terminal.Color(terminal.COLOR_NEVER if args.nocolour 1533 else terminal.COLOR_IF_TERMINAL) 1534 1535 if args.scan_source: 1536 do_scan_source(os.getcwd(), args.update) 1537 return 0 1538 1539 if not any((args.force_sync, args.build_db, args.imply, args.find)): 1540 parser.print_usage() 1541 sys.exit(1) 1542 1543 # prefix the option name with CONFIG_ if missing 1544 configs = [prefix_config(cfg) for cfg in args.configs] 1545 1546 check_top_directory() 1547 1548 if args.imply: 1549 imply_flags = 0 1550 if args.imply_flags == 'all': 1551 imply_flags = -1 1552 1553 elif args.imply_flags: 1554 for flag in args.imply_flags.split(','): 1555 bad = flag not in IMPLY_FLAGS 1556 if bad: 1557 print(f"Invalid flag '{flag}'") 1558 if flag == 'help' or bad: 1559 print("Imply flags: (separate with ',')") 1560 for name, info in IMPLY_FLAGS.items(): 1561 print(f' {name:-15s}: {info[1]}') 1562 parser.print_usage() 1563 sys.exit(1) 1564 imply_flags |= IMPLY_FLAGS[flag][0] 1565 1566 do_imply_config(configs, args.add_imply, imply_flags, args.skip_added) 1567 return 0 1568 1569 if args.find: 1570 do_find_config(configs) 1571 return 0 1572 1573 # We are either building the database or forcing a sync of defconfigs 1574 config_db = {} 1575 db_queue = queue.Queue() 1576 dbt = DatabaseThread(config_db, db_queue) 1577 dbt.daemon = True 1578 dbt.start() 1579 1580 check_clean_directory() 1581 bsettings.setup('') 1582 toolchains = toolchain.Toolchains() 1583 toolchains.GetSettings() 1584 toolchains.Scan(verbose=False) 1585 progress = move_config(toolchains, args, db_queue, col) 1586 db_queue.join() 1587 1588 if args.commit: 1589 subprocess.call(['git', 'add', '-u']) 1590 if configs: 1591 msg = 'Convert %s %sto Kconfig' % (configs[0], 1592 'et al ' if len(configs) > 1 else '') 1593 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % 1594 '\n '.join(configs)) 1595 else: 1596 msg = 'configs: Resync with savedefconfig' 1597 msg += '\n\nRsync all defconfig files using moveconfig.py' 1598 subprocess.call(['git', 'commit', '-s', '-m', msg]) 1599 1600 failed = progress.total - progress.good 1601 failure = f'{failed} failed, ' if failed else '' 1602 if args.build_db: 1603 with open(CONFIG_DATABASE, 'w', encoding='utf-8') as outf: 1604 for defconfig, configs in config_db.items(): 1605 outf.write(f'{defconfig}\n') 1606 for config in sorted(configs.keys()): 1607 outf.write(f' {config}={configs[config]}\n') 1608 outf.write('\n') 1609 print(col.build( 1610 col.RED if failed else col.GREEN, 1611 f'{failure}{len(config_db)} boards written to {CONFIG_DATABASE}')) 1612 else: 1613 if failed: 1614 print(col.build(col.RED, f'{failure}see {FAILED_LIST}', True)) 1615 else: 1616 # Add enough spaces to overwrite the progress indicator 1617 print(col.build( 1618 col.GREEN, f'{progress.total} processed ', bright=True)) 1619 1620 return 0 1621 1622 1623if __name__ == '__main__': 1624 sys.exit(main()) 1625