1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2012 The Chromium OS Authors. 3# 4 5import re 6import glob 7from html.parser import HTMLParser 8import os 9import sys 10import tempfile 11import urllib.request, urllib.error, urllib.parse 12 13from buildman import bsettings 14from u_boot_pylib import command 15from u_boot_pylib import terminal 16from u_boot_pylib import tools 17 18(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH, 19 PRIORITY_CALC) = list(range(4)) 20 21(VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4) 22 23# Simple class to collect links from a page 24class MyHTMLParser(HTMLParser): 25 def __init__(self, arch): 26 """Create a new parser 27 28 After the parser runs, self.links will be set to a list of the links 29 to .xz archives found in the page, and self.arch_link will be set to 30 the one for the given architecture (or None if not found). 31 32 Args: 33 arch: Architecture to search for 34 """ 35 HTMLParser.__init__(self) 36 self.arch_link = None 37 self.links = [] 38 self.re_arch = re.compile('[-_]%s-' % arch) 39 40 def handle_starttag(self, tag, attrs): 41 if tag == 'a': 42 for tag, value in attrs: 43 if tag == 'href': 44 if value and value.endswith('.xz'): 45 self.links.append(value) 46 if self.re_arch.search(value): 47 self.arch_link = value 48 49 50class Toolchain: 51 """A single toolchain 52 53 Public members: 54 gcc: Full path to C compiler 55 path: Directory path containing C compiler 56 cross: Cross compile string, e.g. 'arm-linux-' 57 arch: Architecture of toolchain as determined from the first 58 component of the filename. E.g. arm-linux-gcc becomes arm 59 priority: Toolchain priority (0=highest, 20=lowest) 60 override_toolchain: Toolchain to use for sandbox, overriding the normal 61 one 62 """ 63 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC, 64 arch=None, override_toolchain=None): 65 """Create a new toolchain object. 66 67 Args: 68 fname: Filename of the gcc component 69 test: True to run the toolchain to test it 70 verbose: True to print out the information 71 priority: Priority to use for this toolchain, or PRIORITY_CALC to 72 calculate it 73 """ 74 self.gcc = fname 75 self.path = os.path.dirname(fname) 76 self.override_toolchain = override_toolchain 77 78 # Find the CROSS_COMPILE prefix to use for U-Boot. For example, 79 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. 80 basename = os.path.basename(fname) 81 pos = basename.rfind('-') 82 self.cross = basename[:pos + 1] if pos != -1 else '' 83 84 # The architecture is the first part of the name 85 pos = self.cross.find('-') 86 if arch: 87 self.arch = arch 88 else: 89 self.arch = self.cross[:pos] if pos != -1 else 'sandbox' 90 if self.arch == 'sandbox' and override_toolchain: 91 self.gcc = override_toolchain 92 93 env = self.MakeEnvironment(False) 94 95 # As a basic sanity check, run the C compiler with --version 96 cmd = [fname, '--version'] 97 if priority == PRIORITY_CALC: 98 self.priority = self.GetPriority(fname) 99 else: 100 self.priority = priority 101 if test: 102 result = command.run_pipe([cmd], capture=True, env=env, 103 raise_on_error=False) 104 self.ok = result.return_code == 0 105 if verbose: 106 print('Tool chain test: ', end=' ') 107 if self.ok: 108 print("OK, arch='%s', priority %d" % (self.arch, 109 self.priority)) 110 else: 111 print('BAD') 112 print('Command: ', cmd) 113 print(result.stdout) 114 print(result.stderr) 115 else: 116 self.ok = True 117 118 def GetPriority(self, fname): 119 """Return the priority of the toolchain. 120 121 Toolchains are ranked according to their suitability by their 122 filename prefix. 123 124 Args: 125 fname: Filename of toolchain 126 Returns: 127 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest. 128 """ 129 priority_list = ['-elf', '-unknown-linux-gnu', '-linux', 130 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux', 131 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi', 132 '-linux-gnueabihf', '-le-linux', '-uclinux'] 133 for prio in range(len(priority_list)): 134 if priority_list[prio] in fname: 135 return PRIORITY_CALC + prio 136 return PRIORITY_CALC + prio 137 138 def GetWrapper(self, show_warning=True): 139 """Get toolchain wrapper from the setting file. 140 """ 141 value = '' 142 for name, value in bsettings.get_items('toolchain-wrapper'): 143 if not value: 144 print("Warning: Wrapper not found") 145 if value: 146 value = value + ' ' 147 148 return value 149 150 def GetEnvArgs(self, which): 151 """Get an environment variable/args value based on the the toolchain 152 153 Args: 154 which: VAR_... value to get 155 156 Returns: 157 Value of that environment variable or arguments 158 """ 159 if which == VAR_CROSS_COMPILE: 160 wrapper = self.GetWrapper() 161 base = '' if self.arch == 'sandbox' else self.path 162 return wrapper + os.path.join(base, self.cross) 163 elif which == VAR_PATH: 164 return self.path 165 elif which == VAR_ARCH: 166 return self.arch 167 elif which == VAR_MAKE_ARGS: 168 args = self.MakeArgs() 169 if args: 170 return ' '.join(args) 171 return '' 172 else: 173 raise ValueError('Unknown arg to GetEnvArgs (%d)' % which) 174 175 def MakeEnvironment(self, full_path): 176 """Returns an environment for using the toolchain. 177 178 Thie takes the current environment and adds CROSS_COMPILE so that 179 the tool chain will operate correctly. This also disables localized 180 output and possibly unicode encoded output of all build tools by 181 adding LC_ALL=C. 182 183 Note that os.environb is used to obtain the environment, since in some 184 cases the environment many contain non-ASCII characters and we see 185 errors like: 186 187 UnicodeEncodeError: 'utf-8' codec can't encode characters in position 188 569-570: surrogates not allowed 189 190 Args: 191 full_path: Return the full path in CROSS_COMPILE and don't set 192 PATH 193 Returns: 194 Dict containing the (bytes) environment to use. This is based on the 195 current environment, with changes as needed to CROSS_COMPILE, PATH 196 and LC_ALL. 197 """ 198 env = dict(os.environb) 199 wrapper = self.GetWrapper() 200 201 if self.override_toolchain: 202 # We'll use MakeArgs() to provide this 203 pass 204 elif full_path: 205 env[b'CROSS_COMPILE'] = tools.to_bytes( 206 wrapper + os.path.join(self.path, self.cross)) 207 else: 208 env[b'CROSS_COMPILE'] = tools.to_bytes(wrapper + self.cross) 209 env[b'PATH'] = tools.to_bytes(self.path) + b':' + env[b'PATH'] 210 211 env[b'LC_ALL'] = b'C' 212 213 return env 214 215 def MakeArgs(self): 216 """Create the 'make' arguments for a toolchain 217 218 This is only used when the toolchain is being overridden. Since the 219 U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the 220 environment (and MakeEnvironment()) to override these values. This 221 function returns the arguments to accomplish this. 222 223 Returns: 224 List of arguments to pass to 'make' 225 """ 226 if self.override_toolchain: 227 return ['HOSTCC=%s' % self.override_toolchain, 228 'CC=%s' % self.override_toolchain] 229 return [] 230 231 232class Toolchains: 233 """Manage a list of toolchains for building U-Boot 234 235 We select one toolchain for each architecture type 236 237 Public members: 238 toolchains: Dict of Toolchain objects, keyed by architecture name 239 prefixes: Dict of prefixes to check, keyed by architecture. This can 240 be a full path and toolchain prefix, for example 241 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of 242 something on the search path, for example 243 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported. 244 paths: List of paths to check for toolchains (may contain wildcards) 245 """ 246 247 def __init__(self, override_toolchain=None): 248 self.toolchains = {} 249 self.prefixes = {} 250 self.paths = [] 251 self.override_toolchain = override_toolchain 252 self._make_flags = dict(bsettings.get_items('make-flags')) 253 254 def GetPathList(self, show_warning=True): 255 """Get a list of available toolchain paths 256 257 Args: 258 show_warning: True to show a warning if there are no tool chains. 259 260 Returns: 261 List of strings, each a path to a toolchain mentioned in the 262 [toolchain] section of the settings file. 263 """ 264 toolchains = bsettings.get_items('toolchain') 265 if show_warning and not toolchains: 266 print(("Warning: No tool chains. Please run 'buildman " 267 "--fetch-arch all' to download all available toolchains, or " 268 "add a [toolchain] section to your buildman config file " 269 "%s. See buildman.rst for details" % 270 bsettings.config_fname)) 271 272 paths = [] 273 for name, value in toolchains: 274 if '*' in value: 275 paths += glob.glob(value) 276 else: 277 paths.append(value) 278 return paths 279 280 def GetSettings(self, show_warning=True): 281 """Get toolchain settings from the settings file. 282 283 Args: 284 show_warning: True to show a warning if there are no tool chains. 285 """ 286 self.prefixes = bsettings.get_items('toolchain-prefix') 287 self.paths += self.GetPathList(show_warning) 288 289 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC, 290 arch=None): 291 """Add a toolchain to our list 292 293 We select the given toolchain as our preferred one for its 294 architecture if it is a higher priority than the others. 295 296 Args: 297 fname: Filename of toolchain's gcc driver 298 test: True to run the toolchain to test it 299 priority: Priority to use for this toolchain 300 arch: Toolchain architecture, or None if not known 301 """ 302 toolchain = Toolchain(fname, test, verbose, priority, arch, 303 self.override_toolchain) 304 add_it = toolchain.ok 305 if toolchain.arch in self.toolchains: 306 add_it = (toolchain.priority < 307 self.toolchains[toolchain.arch].priority) 308 if add_it: 309 self.toolchains[toolchain.arch] = toolchain 310 elif verbose: 311 print(("Toolchain '%s' at priority %d will be ignored because " 312 "another toolchain for arch '%s' has priority %d" % 313 (toolchain.gcc, toolchain.priority, toolchain.arch, 314 self.toolchains[toolchain.arch].priority))) 315 316 def ScanPath(self, path, verbose): 317 """Scan a path for a valid toolchain 318 319 Args: 320 path: Path to scan 321 verbose: True to print out progress information 322 Returns: 323 Filename of C compiler if found, else None 324 """ 325 fnames = [] 326 for subdir in ['.', 'bin', 'usr/bin']: 327 dirname = os.path.join(path, subdir) 328 if verbose: print(" - looking in '%s'" % dirname) 329 for fname in glob.glob(dirname + '/*gcc'): 330 if verbose: print(" - found '%s'" % fname) 331 fnames.append(fname) 332 return fnames 333 334 def ScanPathEnv(self, fname): 335 """Scan the PATH environment variable for a given filename. 336 337 Args: 338 fname: Filename to scan for 339 Returns: 340 List of matching pathanames, or [] if none 341 """ 342 pathname_list = [] 343 for path in os.environ["PATH"].split(os.pathsep): 344 path = path.strip('"') 345 pathname = os.path.join(path, fname) 346 if os.path.exists(pathname): 347 pathname_list.append(pathname) 348 return pathname_list 349 350 def Scan(self, verbose): 351 """Scan for available toolchains and select the best for each arch. 352 353 We look for all the toolchains we can file, figure out the 354 architecture for each, and whether it works. Then we select the 355 highest priority toolchain for each arch. 356 357 Args: 358 verbose: True to print out progress information 359 """ 360 if verbose: print('Scanning for tool chains') 361 for name, value in self.prefixes: 362 if verbose: print(" - scanning prefix '%s'" % value) 363 if os.path.exists(value): 364 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name) 365 continue 366 fname = value + 'gcc' 367 if os.path.exists(fname): 368 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name) 369 continue 370 fname_list = self.ScanPathEnv(fname) 371 for f in fname_list: 372 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name) 373 if not fname_list: 374 raise ValueError("No tool chain found for prefix '%s'" % 375 value) 376 for path in self.paths: 377 if verbose: print(" - scanning path '%s'" % path) 378 fnames = self.ScanPath(path, verbose) 379 for fname in fnames: 380 self.Add(fname, True, verbose) 381 382 def List(self): 383 """List out the selected toolchains for each architecture""" 384 col = terminal.Color() 385 print(col.build(col.BLUE, 'List of available toolchains (%d):' % 386 len(self.toolchains))) 387 if len(self.toolchains): 388 for key, value in sorted(self.toolchains.items()): 389 print('%-10s: %s' % (key, value.gcc)) 390 else: 391 print('None') 392 393 def Select(self, arch): 394 """Returns the toolchain for a given architecture 395 396 Args: 397 args: Name of architecture (e.g. 'arm', 'ppc_8xx') 398 399 returns: 400 toolchain object, or None if none found 401 """ 402 for tag, value in bsettings.get_items('toolchain-alias'): 403 if arch == tag: 404 for alias in value.split(): 405 if alias in self.toolchains: 406 return self.toolchains[alias] 407 408 if not arch in self.toolchains: 409 raise ValueError("No tool chain found for arch '%s'" % arch) 410 return self.toolchains[arch] 411 412 def ResolveReferences(self, var_dict, args): 413 """Resolve variable references in a string 414 415 This converts ${blah} within the string to the value of blah. 416 This function works recursively. 417 418 Args: 419 var_dict: Dictionary containing variables and their values 420 args: String containing make arguments 421 Returns: 422 Resolved string 423 424 >>> bsettings.setup(None) 425 >>> tcs = Toolchains() 426 >>> tcs.Add('fred', False) 427 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \ 428 'second' : '2nd'} 429 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set') 430 'this=OBLIQUE_set' 431 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd') 432 'this=OBLIQUE_setfi2ndrstnd' 433 """ 434 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})') 435 436 while True: 437 m = re_var.search(args) 438 if not m: 439 break 440 lookup = m.group(0)[2:-1] 441 value = var_dict.get(lookup, '') 442 args = args[:m.start(0)] + value + args[m.end(0):] 443 return args 444 445 def GetMakeArguments(self, brd): 446 """Returns 'make' arguments for a given board 447 448 The flags are in a section called 'make-flags'. Flags are named 449 after the target they represent, for example snapper9260=TESTING=1 450 will pass TESTING=1 to make when building the snapper9260 board. 451 452 References to other boards can be added in the string also. For 453 example: 454 455 [make-flags] 456 at91-boards=ENABLE_AT91_TEST=1 457 snapper9260=${at91-boards} BUILD_TAG=442 458 snapper9g45=${at91-boards} BUILD_TAG=443 459 460 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 461 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. 462 463 A special 'target' variable is set to the board target. 464 465 Args: 466 brd: Board object for the board to check. 467 Returns: 468 'make' flags for that board, or '' if none 469 """ 470 self._make_flags['target'] = brd.target 471 arg_str = self.ResolveReferences(self._make_flags, 472 self._make_flags.get(brd.target, '')) 473 args = re.findall("(?:\".*?\"|\S)+", arg_str) 474 i = 0 475 while i < len(args): 476 args[i] = args[i].replace('"', '') 477 if not args[i]: 478 del args[i] 479 else: 480 i += 1 481 return args 482 483 def LocateArchUrl(self, fetch_arch): 484 """Find a toolchain available online 485 486 Look in standard places for available toolchains. At present the 487 only standard place is at kernel.org. 488 489 Args: 490 arch: Architecture to look for, or 'list' for all 491 Returns: 492 If fetch_arch is 'list', a tuple: 493 Machine architecture (e.g. x86_64) 494 List of toolchains 495 else 496 URL containing this toolchain, if avaialble, else None 497 """ 498 arch = command.output_one_line('uname', '-m') 499 if arch == 'aarch64': 500 arch = 'arm64' 501 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' 502 versions = ['13.2.0', '12.2.0'] 503 links = [] 504 for version in versions: 505 url = '%s/%s/%s/' % (base, arch, version) 506 print('Checking: %s' % url) 507 response = urllib.request.urlopen(url) 508 html = tools.to_string(response.read()) 509 parser = MyHTMLParser(fetch_arch) 510 parser.feed(html) 511 if fetch_arch == 'list': 512 links += parser.links 513 elif parser.arch_link: 514 return url + parser.arch_link 515 if fetch_arch == 'list': 516 return arch, links 517 return None 518 519 def Unpack(self, fname, dest): 520 """Unpack a tar file 521 522 Args: 523 fname: Filename to unpack 524 dest: Destination directory 525 Returns: 526 Directory name of the first entry in the archive, without the 527 trailing / 528 """ 529 stdout = command.output('tar', 'xvfJ', fname, '-C', dest) 530 dirs = stdout.splitlines()[1].split('/')[:2] 531 return '/'.join(dirs) 532 533 def TestSettingsHasPath(self, path): 534 """Check if buildman will find this toolchain 535 536 Returns: 537 True if the path is in settings, False if not 538 """ 539 paths = self.GetPathList(False) 540 return path in paths 541 542 def ListArchs(self): 543 """List architectures with available toolchains to download""" 544 host_arch, archives = self.LocateArchUrl('list') 545 re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*') 546 arch_set = set() 547 for archive in archives: 548 # Remove the host architecture from the start 549 arch = re_arch.match(archive[len(host_arch):]) 550 if arch: 551 if arch.group(1) != '2.0' and arch.group(1) != '64': 552 arch_set.add(arch.group(1)) 553 return sorted(arch_set) 554 555 def FetchAndInstall(self, arch): 556 """Fetch and install a new toolchain 557 558 arch: 559 Architecture to fetch, or 'list' to list 560 """ 561 # Fist get the URL for this architecture 562 col = terminal.Color() 563 print(col.build(col.BLUE, "Downloading toolchain for arch '%s'" % arch)) 564 url = self.LocateArchUrl(arch) 565 if not url: 566 print(("Cannot find toolchain for arch '%s' - use 'list' to list" % 567 arch)) 568 return 2 569 home = os.environ['HOME'] 570 dest = os.path.join(home, '.buildman-toolchains') 571 if not os.path.exists(dest): 572 os.mkdir(dest) 573 574 # Download the tar file for this toolchain and unpack it 575 tarfile, tmpdir = tools.download(url, '.buildman') 576 if not tarfile: 577 return 1 578 print(col.build(col.GREEN, 'Unpacking to: %s' % dest), end=' ') 579 sys.stdout.flush() 580 path = self.Unpack(tarfile, dest) 581 os.remove(tarfile) 582 os.rmdir(tmpdir) 583 print() 584 585 # Check that the toolchain works 586 print(col.build(col.GREEN, 'Testing')) 587 dirpath = os.path.join(dest, path) 588 compiler_fname_list = self.ScanPath(dirpath, True) 589 if not compiler_fname_list: 590 print('Could not locate C compiler - fetch failed.') 591 return 1 592 if len(compiler_fname_list) != 1: 593 print(col.build(col.RED, 'Warning, ambiguous toolchains: %s' % 594 ', '.join(compiler_fname_list))) 595 toolchain = Toolchain(compiler_fname_list[0], True, True) 596 597 # Make sure that it will be found by buildman 598 if not self.TestSettingsHasPath(dirpath): 599 print(("Adding 'download' to config file '%s'" % 600 bsettings.config_fname)) 601 bsettings.set_item('toolchain', 'download', '%s/*/*' % dest) 602 return 0 603