1# SPDX-License-Identifier: GPL-2.0+
2# Copyright (c) 2012 The Chromium OS Authors.
3#
4
5import re
6import glob
7from html.parser import HTMLParser
8import os
9import sys
10import tempfile
11import urllib.request, urllib.error, urllib.parse
12
13from buildman import bsettings
14from u_boot_pylib import command
15from u_boot_pylib import terminal
16from u_boot_pylib import tools
17
18(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
19    PRIORITY_CALC) = list(range(4))
20
21(VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4)
22
23# Simple class to collect links from a page
24class MyHTMLParser(HTMLParser):
25    def __init__(self, arch):
26        """Create a new parser
27
28        After the parser runs, self.links will be set to a list of the links
29        to .xz archives found in the page, and self.arch_link will be set to
30        the one for the given architecture (or None if not found).
31
32        Args:
33            arch: Architecture to search for
34        """
35        HTMLParser.__init__(self)
36        self.arch_link = None
37        self.links = []
38        self.re_arch = re.compile('[-_]%s-' % arch)
39
40    def handle_starttag(self, tag, attrs):
41        if tag == 'a':
42            for tag, value in attrs:
43                if tag == 'href':
44                    if value and value.endswith('.xz'):
45                        self.links.append(value)
46                        if self.re_arch.search(value):
47                            self.arch_link = value
48
49
50class Toolchain:
51    """A single toolchain
52
53    Public members:
54        gcc: Full path to C compiler
55        path: Directory path containing C compiler
56        cross: Cross compile string, e.g. 'arm-linux-'
57        arch: Architecture of toolchain as determined from the first
58                component of the filename. E.g. arm-linux-gcc becomes arm
59        priority: Toolchain priority (0=highest, 20=lowest)
60        override_toolchain: Toolchain to use for sandbox, overriding the normal
61                one
62    """
63    def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
64                 arch=None, override_toolchain=None):
65        """Create a new toolchain object.
66
67        Args:
68            fname: Filename of the gcc component
69            test: True to run the toolchain to test it
70            verbose: True to print out the information
71            priority: Priority to use for this toolchain, or PRIORITY_CALC to
72                calculate it
73        """
74        self.gcc = fname
75        self.path = os.path.dirname(fname)
76        self.override_toolchain = override_toolchain
77
78        # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
79        # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
80        basename = os.path.basename(fname)
81        pos = basename.rfind('-')
82        self.cross = basename[:pos + 1] if pos != -1 else ''
83
84        # The architecture is the first part of the name
85        pos = self.cross.find('-')
86        if arch:
87            self.arch = arch
88        else:
89            self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
90        if self.arch == 'sandbox' and override_toolchain:
91            self.gcc = override_toolchain
92
93        env = self.MakeEnvironment(False)
94
95        # As a basic sanity check, run the C compiler with --version
96        cmd = [fname, '--version']
97        if priority == PRIORITY_CALC:
98            self.priority = self.GetPriority(fname)
99        else:
100            self.priority = priority
101        if test:
102            result = command.run_pipe([cmd], capture=True, env=env,
103                                     raise_on_error=False)
104            self.ok = result.return_code == 0
105            if verbose:
106                print('Tool chain test: ', end=' ')
107                if self.ok:
108                    print("OK, arch='%s', priority %d" % (self.arch,
109                                                          self.priority))
110                else:
111                    print('BAD')
112                    print('Command: ', cmd)
113                    print(result.stdout)
114                    print(result.stderr)
115        else:
116            self.ok = True
117
118    def GetPriority(self, fname):
119        """Return the priority of the toolchain.
120
121        Toolchains are ranked according to their suitability by their
122        filename prefix.
123
124        Args:
125            fname: Filename of toolchain
126        Returns:
127            Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
128        """
129        priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
130            '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
131            '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
132            '-linux-gnueabihf', '-le-linux', '-uclinux']
133        for prio in range(len(priority_list)):
134            if priority_list[prio] in fname:
135                return PRIORITY_CALC + prio
136        return PRIORITY_CALC + prio
137
138    def GetWrapper(self, show_warning=True):
139        """Get toolchain wrapper from the setting file.
140        """
141        value = ''
142        for name, value in bsettings.get_items('toolchain-wrapper'):
143            if not value:
144                print("Warning: Wrapper not found")
145        if value:
146            value = value + ' '
147
148        return value
149
150    def GetEnvArgs(self, which):
151        """Get an environment variable/args value based on the the toolchain
152
153        Args:
154            which: VAR_... value to get
155
156        Returns:
157            Value of that environment variable or arguments
158        """
159        if which == VAR_CROSS_COMPILE:
160            wrapper = self.GetWrapper()
161            base = '' if self.arch == 'sandbox' else self.path
162            return wrapper + os.path.join(base, self.cross)
163        elif which == VAR_PATH:
164            return self.path
165        elif which == VAR_ARCH:
166            return self.arch
167        elif which == VAR_MAKE_ARGS:
168            args = self.MakeArgs()
169            if args:
170                return ' '.join(args)
171            return ''
172        else:
173            raise ValueError('Unknown arg to GetEnvArgs (%d)' % which)
174
175    def MakeEnvironment(self, full_path):
176        """Returns an environment for using the toolchain.
177
178        Thie takes the current environment and adds CROSS_COMPILE so that
179        the tool chain will operate correctly. This also disables localized
180        output and possibly unicode encoded output of all build tools by
181        adding LC_ALL=C.
182
183        Note that os.environb is used to obtain the environment, since in some
184        cases the environment many contain non-ASCII characters and we see
185        errors like:
186
187          UnicodeEncodeError: 'utf-8' codec can't encode characters in position
188             569-570: surrogates not allowed
189
190        Args:
191            full_path: Return the full path in CROSS_COMPILE and don't set
192                PATH
193        Returns:
194            Dict containing the (bytes) environment to use. This is based on the
195            current environment, with changes as needed to CROSS_COMPILE, PATH
196            and LC_ALL.
197        """
198        env = dict(os.environb)
199        wrapper = self.GetWrapper()
200
201        if self.override_toolchain:
202            # We'll use MakeArgs() to provide this
203            pass
204        elif full_path:
205            env[b'CROSS_COMPILE'] = tools.to_bytes(
206                wrapper + os.path.join(self.path, self.cross))
207        else:
208            env[b'CROSS_COMPILE'] = tools.to_bytes(wrapper + self.cross)
209            env[b'PATH'] = tools.to_bytes(self.path) + b':' + env[b'PATH']
210
211        env[b'LC_ALL'] = b'C'
212
213        return env
214
215    def MakeArgs(self):
216        """Create the 'make' arguments for a toolchain
217
218        This is only used when the toolchain is being overridden. Since the
219        U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the
220        environment (and MakeEnvironment()) to override these values. This
221        function returns the arguments to accomplish this.
222
223        Returns:
224            List of arguments to pass to 'make'
225        """
226        if self.override_toolchain:
227            return ['HOSTCC=%s' % self.override_toolchain,
228                    'CC=%s' % self.override_toolchain]
229        return []
230
231
232class Toolchains:
233    """Manage a list of toolchains for building U-Boot
234
235    We select one toolchain for each architecture type
236
237    Public members:
238        toolchains: Dict of Toolchain objects, keyed by architecture name
239        prefixes: Dict of prefixes to check, keyed by architecture. This can
240            be a full path and toolchain prefix, for example
241            {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
242            something on the search path, for example
243            {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
244        paths: List of paths to check for toolchains (may contain wildcards)
245    """
246
247    def __init__(self, override_toolchain=None):
248        self.toolchains = {}
249        self.prefixes = {}
250        self.paths = []
251        self.override_toolchain = override_toolchain
252        self._make_flags = dict(bsettings.get_items('make-flags'))
253
254    def GetPathList(self, show_warning=True):
255        """Get a list of available toolchain paths
256
257        Args:
258            show_warning: True to show a warning if there are no tool chains.
259
260        Returns:
261            List of strings, each a path to a toolchain mentioned in the
262            [toolchain] section of the settings file.
263        """
264        toolchains = bsettings.get_items('toolchain')
265        if show_warning and not toolchains:
266            print(("Warning: No tool chains. Please run 'buildman "
267                   "--fetch-arch all' to download all available toolchains, or "
268                   "add a [toolchain] section to your buildman config file "
269                   "%s. See buildman.rst for details" %
270                   bsettings.config_fname))
271
272        paths = []
273        for name, value in toolchains:
274            if '*' in value:
275                paths += glob.glob(value)
276            else:
277                paths.append(value)
278        return paths
279
280    def GetSettings(self, show_warning=True):
281        """Get toolchain settings from the settings file.
282
283        Args:
284            show_warning: True to show a warning if there are no tool chains.
285        """
286        self.prefixes = bsettings.get_items('toolchain-prefix')
287        self.paths += self.GetPathList(show_warning)
288
289    def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
290            arch=None):
291        """Add a toolchain to our list
292
293        We select the given toolchain as our preferred one for its
294        architecture if it is a higher priority than the others.
295
296        Args:
297            fname: Filename of toolchain's gcc driver
298            test: True to run the toolchain to test it
299            priority: Priority to use for this toolchain
300            arch: Toolchain architecture, or None if not known
301        """
302        toolchain = Toolchain(fname, test, verbose, priority, arch,
303                              self.override_toolchain)
304        add_it = toolchain.ok
305        if toolchain.arch in self.toolchains:
306            add_it = (toolchain.priority <
307                        self.toolchains[toolchain.arch].priority)
308        if add_it:
309            self.toolchains[toolchain.arch] = toolchain
310        elif verbose:
311            print(("Toolchain '%s' at priority %d will be ignored because "
312                   "another toolchain for arch '%s' has priority %d" %
313                   (toolchain.gcc, toolchain.priority, toolchain.arch,
314                    self.toolchains[toolchain.arch].priority)))
315
316    def ScanPath(self, path, verbose):
317        """Scan a path for a valid toolchain
318
319        Args:
320            path: Path to scan
321            verbose: True to print out progress information
322        Returns:
323            Filename of C compiler if found, else None
324        """
325        fnames = []
326        for subdir in ['.', 'bin', 'usr/bin']:
327            dirname = os.path.join(path, subdir)
328            if verbose: print("      - looking in '%s'" % dirname)
329            for fname in glob.glob(dirname + '/*gcc'):
330                if verbose: print("         - found '%s'" % fname)
331                fnames.append(fname)
332        return fnames
333
334    def ScanPathEnv(self, fname):
335        """Scan the PATH environment variable for a given filename.
336
337        Args:
338            fname: Filename to scan for
339        Returns:
340            List of matching pathanames, or [] if none
341        """
342        pathname_list = []
343        for path in os.environ["PATH"].split(os.pathsep):
344            path = path.strip('"')
345            pathname = os.path.join(path, fname)
346            if os.path.exists(pathname):
347                pathname_list.append(pathname)
348        return pathname_list
349
350    def Scan(self, verbose):
351        """Scan for available toolchains and select the best for each arch.
352
353        We look for all the toolchains we can file, figure out the
354        architecture for each, and whether it works. Then we select the
355        highest priority toolchain for each arch.
356
357        Args:
358            verbose: True to print out progress information
359        """
360        if verbose: print('Scanning for tool chains')
361        for name, value in self.prefixes:
362            if verbose: print("   - scanning prefix '%s'" % value)
363            if os.path.exists(value):
364                self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
365                continue
366            fname = value + 'gcc'
367            if os.path.exists(fname):
368                self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
369                continue
370            fname_list = self.ScanPathEnv(fname)
371            for f in fname_list:
372                self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
373            if not fname_list:
374                raise ValueError("No tool chain found for prefix '%s'" %
375                                   value)
376        for path in self.paths:
377            if verbose: print("   - scanning path '%s'" % path)
378            fnames = self.ScanPath(path, verbose)
379            for fname in fnames:
380                self.Add(fname, True, verbose)
381
382    def List(self):
383        """List out the selected toolchains for each architecture"""
384        col = terminal.Color()
385        print(col.build(col.BLUE, 'List of available toolchains (%d):' %
386                        len(self.toolchains)))
387        if len(self.toolchains):
388            for key, value in sorted(self.toolchains.items()):
389                print('%-10s: %s' % (key, value.gcc))
390        else:
391            print('None')
392
393    def Select(self, arch):
394        """Returns the toolchain for a given architecture
395
396        Args:
397            args: Name of architecture (e.g. 'arm', 'ppc_8xx')
398
399        returns:
400            toolchain object, or None if none found
401        """
402        for tag, value in bsettings.get_items('toolchain-alias'):
403            if arch == tag:
404                for alias in value.split():
405                    if alias in self.toolchains:
406                        return self.toolchains[alias]
407
408        if not arch in self.toolchains:
409            raise ValueError("No tool chain found for arch '%s'" % arch)
410        return self.toolchains[arch]
411
412    def ResolveReferences(self, var_dict, args):
413        """Resolve variable references in a string
414
415        This converts ${blah} within the string to the value of blah.
416        This function works recursively.
417
418        Args:
419            var_dict: Dictionary containing variables and their values
420            args: String containing make arguments
421        Returns:
422            Resolved string
423
424        >>> bsettings.setup(None)
425        >>> tcs = Toolchains()
426        >>> tcs.Add('fred', False)
427        >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
428                        'second' : '2nd'}
429        >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
430        'this=OBLIQUE_set'
431        >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
432        'this=OBLIQUE_setfi2ndrstnd'
433        """
434        re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
435
436        while True:
437            m = re_var.search(args)
438            if not m:
439                break
440            lookup = m.group(0)[2:-1]
441            value = var_dict.get(lookup, '')
442            args = args[:m.start(0)] + value + args[m.end(0):]
443        return args
444
445    def GetMakeArguments(self, brd):
446        """Returns 'make' arguments for a given board
447
448        The flags are in a section called 'make-flags'. Flags are named
449        after the target they represent, for example snapper9260=TESTING=1
450        will pass TESTING=1 to make when building the snapper9260 board.
451
452        References to other boards can be added in the string also. For
453        example:
454
455        [make-flags]
456        at91-boards=ENABLE_AT91_TEST=1
457        snapper9260=${at91-boards} BUILD_TAG=442
458        snapper9g45=${at91-boards} BUILD_TAG=443
459
460        This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
461        and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
462
463        A special 'target' variable is set to the board target.
464
465        Args:
466            brd: Board object for the board to check.
467        Returns:
468            'make' flags for that board, or '' if none
469        """
470        self._make_flags['target'] = brd.target
471        arg_str = self.ResolveReferences(self._make_flags,
472                           self._make_flags.get(brd.target, ''))
473        args = re.findall("(?:\".*?\"|\S)+", arg_str)
474        i = 0
475        while i < len(args):
476            args[i] = args[i].replace('"', '')
477            if not args[i]:
478                del args[i]
479            else:
480                i += 1
481        return args
482
483    def LocateArchUrl(self, fetch_arch):
484        """Find a toolchain available online
485
486        Look in standard places for available toolchains. At present the
487        only standard place is at kernel.org.
488
489        Args:
490            arch: Architecture to look for, or 'list' for all
491        Returns:
492            If fetch_arch is 'list', a tuple:
493                Machine architecture (e.g. x86_64)
494                List of toolchains
495            else
496                URL containing this toolchain, if avaialble, else None
497        """
498        arch = command.output_one_line('uname', '-m')
499        if arch == 'aarch64':
500            arch = 'arm64'
501        base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
502        versions = ['13.2.0', '12.2.0']
503        links = []
504        for version in versions:
505            url = '%s/%s/%s/' % (base, arch, version)
506            print('Checking: %s' % url)
507            response = urllib.request.urlopen(url)
508            html = tools.to_string(response.read())
509            parser = MyHTMLParser(fetch_arch)
510            parser.feed(html)
511            if fetch_arch == 'list':
512                links += parser.links
513            elif parser.arch_link:
514                return url + parser.arch_link
515        if fetch_arch == 'list':
516            return arch, links
517        return None
518
519    def Unpack(self, fname, dest):
520        """Unpack a tar file
521
522        Args:
523            fname: Filename to unpack
524            dest: Destination directory
525        Returns:
526            Directory name of the first entry in the archive, without the
527            trailing /
528        """
529        stdout = command.output('tar', 'xvfJ', fname, '-C', dest)
530        dirs = stdout.splitlines()[1].split('/')[:2]
531        return '/'.join(dirs)
532
533    def TestSettingsHasPath(self, path):
534        """Check if buildman will find this toolchain
535
536        Returns:
537            True if the path is in settings, False if not
538        """
539        paths = self.GetPathList(False)
540        return path in paths
541
542    def ListArchs(self):
543        """List architectures with available toolchains to download"""
544        host_arch, archives = self.LocateArchUrl('list')
545        re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*')
546        arch_set = set()
547        for archive in archives:
548            # Remove the host architecture from the start
549            arch = re_arch.match(archive[len(host_arch):])
550            if arch:
551                if arch.group(1) != '2.0' and arch.group(1) != '64':
552                    arch_set.add(arch.group(1))
553        return sorted(arch_set)
554
555    def FetchAndInstall(self, arch):
556        """Fetch and install a new toolchain
557
558        arch:
559            Architecture to fetch, or 'list' to list
560        """
561        # Fist get the URL for this architecture
562        col = terminal.Color()
563        print(col.build(col.BLUE, "Downloading toolchain for arch '%s'" % arch))
564        url = self.LocateArchUrl(arch)
565        if not url:
566            print(("Cannot find toolchain for arch '%s' - use 'list' to list" %
567                   arch))
568            return 2
569        home = os.environ['HOME']
570        dest = os.path.join(home, '.buildman-toolchains')
571        if not os.path.exists(dest):
572            os.mkdir(dest)
573
574        # Download the tar file for this toolchain and unpack it
575        tarfile, tmpdir = tools.download(url, '.buildman')
576        if not tarfile:
577            return 1
578        print(col.build(col.GREEN, 'Unpacking to: %s' % dest), end=' ')
579        sys.stdout.flush()
580        path = self.Unpack(tarfile, dest)
581        os.remove(tarfile)
582        os.rmdir(tmpdir)
583        print()
584
585        # Check that the toolchain works
586        print(col.build(col.GREEN, 'Testing'))
587        dirpath = os.path.join(dest, path)
588        compiler_fname_list = self.ScanPath(dirpath, True)
589        if not compiler_fname_list:
590            print('Could not locate C compiler - fetch failed.')
591            return 1
592        if len(compiler_fname_list) != 1:
593            print(col.build(col.RED, 'Warning, ambiguous toolchains: %s' %
594                            ', '.join(compiler_fname_list)))
595        toolchain = Toolchain(compiler_fname_list[0], True, True)
596
597        # Make sure that it will be found by buildman
598        if not self.TestSettingsHasPath(dirpath):
599            print(("Adding 'download' to config file '%s'" %
600                   bsettings.config_fname))
601            bsettings.set_item('toolchain', 'download', '%s/*/*' % dest)
602        return 0
603