1#!/usr/bin/env bash 2 3# Copyright 2018 The Fuchsia Authors 4# 5# Use of this source code is governed by a MIT-style 6# license that can be found in the LICENSE file or at 7# https://opensource.org/licenses/MIT 8 9readonly SCRIPT_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" 10readonly ZIRCON_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" 11readonly PREBUILTS_DIR="$(cd "${ZIRCON_ROOT}/prebuilt" && pwd)" 12readonly DOWNLOAD_DIR="${PREBUILTS_DIR}/downloads" 13readonly ENSURE_FILE="${PREBUILTS_DIR}/zircon.ensure" 14readonly VERSIONS_FILE="${PREBUILTS_DIR}/zircon.versions" 15readonly URL_PREFIX="https://storage.googleapis.com/fuchsia" 16 17# This script assumes that ENSURE_FILE and VERSIONS_FILE match up. 18# The update-prebuilt-versions script ensures that they do. When a 19# cipd binary is available, ENSURE_FILE controls the downloads. 20# Otherwise, VERSIONS_FILE controls the downloads. In both cases we 21# write $DOWNLOAD_DIR/$PACKAGE.stamp files with the versions from 22# VERSIONS_FILE on faith that that's what we just unpacked. 23 24set -o pipefail 25 26cipd_ok=true 27case "$#:$1" in 280:) 29 mode=update 30 ;; 311:--verify) 32 mode=verify 33 ;; 341:--list) 35 mode=list 36 ;; 371:--no-cipd) 38 mode=update 39 cipd_ok=false 40 ;; 41*) 42 echo >&2 "Usage: $0 [--verify|--list|--no-cipd]" 43 exit 1 44 ;; 45esac 46readonly cipd_ok 47 48case "$(uname)-$(uname -m)" in 49Darwin-x86_64) 50 PLATFORM=mac-amd64 51 ;; 52Linux-x86_64) 53 PLATFORM=linux-amd64 54 ;; 55Linux-aarch64) 56 PLATFORM=linux-arm64 57 ;; 58*) 59 echo 'Unknown operating system.' 60 exit 1 61 ;; 62esac 63readonly PLATFORM 64 65update_stamp() { 66 local -r package="$1" version="$2" download_file="$3" 67 local -r stamp="${DOWNLOAD_DIR}/${download_file%.*}.stamp" 68 mkdir -p "$(dirname "$stamp")" && echo "$version" > "$stamp" 69} 70 71verify_stamp() { 72 local verbose=false 73 if [[ "$1" = "--verbose" ]]; then 74 verbose=true 75 shift 76 fi 77 local -r package="$1" version="$2" download_file="$3" 78 local -r stamp="${DOWNLOAD_DIR}/${download_file%.*}.stamp" 79 local stamp_version 80 if [[ -r "$stamp" ]]; then 81 stamp_version="$(< "$stamp")" 82 else 83 stamp_version="missing" 84 fi 85 if [[ "$stamp_version" = "$version" ]]; then 86 return 0 87 fi 88 if $verbose; then 89 echo "WARNING: unpacked $package $stamp_version != current $version" 90 fi 91 return 1 92} 93 94list_package() { 95 local -r package="$1" version="$2" download_file="$3" single_file="$4" 96 local -r package_suffix="$5" 97 local -r stamp="${DOWNLOAD_DIR}/${download_file%.*}.stamp" 98 local stamp_version 99 if [[ -r "$stamp" ]]; then 100 stamp_version="$(< "$stamp")" 101 else 102 stamp_version="missing" 103 fi 104 local installed="${stamp_version:-missing}" 105 if [[ "$installed" = "$version" ]]; then 106 installed="current" 107 fi 108 echo "$package$package_suffix" "installed=$installed" "current=$version" 109} 110 111verify_file() { 112 local -r file="$1" 113 local -r sum="$2" 114 shasum --binary --check --status <<EOF 115$sum *$file 116EOF 117} 118 119download_package() { 120 local -r package="$1" version="$2" download_file="$3" single_file="$4" 121 local -r package_suffix="$5" 122 local -r url="${URL_PREFIX}/${package}${package_suffix}/${version}" 123 local -r download_file_with_dir="${DOWNLOAD_DIR}/${download_file}" 124 125 # If the stamp file says it's already in place, do nothing more. 126 verify_stamp "$package" "$version" "$download_file" && return 127 128 rm -f -- "$download_file_with_dir" 129 echo "Downloading $url" 130 curl --progress-bar -continue-at=- --location \ 131 --create-dirs --output "$download_file_with_dir" "$url" || return 132 133 verify_file "$download_file_with_dir" "$version" || { 134 echo >&2 "*** VERIFICATION ERROR ***" 135 echo >&2 "*** VERIFICATION ERROR *** $download_file from $url" 136 echo >&2 "*** VERIFICATION ERROR *** Not using the file!" 137 return 1 138 } 139 140 echo "Unpacking $download_file" 141 if $single_file; then 142 # The archive contains .cipd* metadata files and a single real file 143 # whose name is the same as the basename of the package. 144 unzip -q -o -d "$DOWNLOAD_DIR" "$download_file_with_dir" \ 145 "${package##*/}" || return 146 else 147 local -r dir="${download_file_with_dir%.zip}" 148 rm -rf -- "$dir" 149 unzip -q -d "$dir" "$download_file_with_dir" || return 150 fi 151 152 update_stamp "$package" "$version" "$download_file" 153} 154 155for_each_package() { 156 local package version download_file single_file package_suffix status=0 157 while read package version; do 158 case "$package" in 159 ''|\#*) 160 continue 161 ;; 162 tools/*/${PLATFORM}) 163 # These are standalone executables contained in a .zip file. 164 package="${package%/*}" 165 download_file="${package#tools/}.zip" 166 single_file=true 167 package_suffix="/${PLATFORM}" 168 ;; 169 */${PLATFORM}) 170 package="${package%/*}" 171 # Subdirectories are packed in .zip files. 172 download_file="${package}.zip" 173 single_file=false 174 package_suffix="/${PLATFORM}" 175 ;; 176 firmware/*) 177 # These are the same for every host platform. 178 download_file="${package}.zip" 179 single_file=false 180 package_suffix='' 181 ;; 182 *) 183 # Skip lines for other platforms. 184 continue 185 ;; 186 esac 187 "$@" "$package" "$version" "$download_file" $single_file "$package_suffix" || 188 status=$? 189 done 190 return $status 191} 192 193find_cipd() { 194 # If the Zircon checkout is part of a jiri checkout that includes 195 # //buildtools, then find cipd there. Otherwise, if cipd is in 196 # the PATH, take it from there. 197 type -p "${ZIRCON_ROOT}/../buildtools/cipd" || type -p cipd 198} 199 200update_via_cipd() { 201 local -r CIPD="$1" internal_access="$2" 202 203 local -a ensure_files=("${PREBUILTS_DIR}/zircon.ensure") 204 if $internal_access; then 205 ensure_files+=("${PREBUILTS_DIR}/zircon_internal.ensure") 206 fi 207 208 (sed '/^\$/!d' "${ensure_files[@]}" && sed '/^\$/d' "${ensure_files[@]}") | 209 "$CIPD" ensure -ensure-file - -root "$DOWNLOAD_DIR" -log-level warning 210 rc=$? 211 if [[ $rc -ne 0 ]]; then 212 echo >&2 "$0: $CIPD failed. For direct downloads, remove cipd from PATH." 213 return $rc 214 fi 215 216 "$(dirname $0)/update-prebuilt-versions" --verify || { 217 echo >&2 "$0: update-prebuilt-versions not run after ensure file changed!" 218 return 2 219 } 220 221 # Now update the .stamp files so that --verify will be happy. 222 for_each_package update_stamp < "$VERSIONS_FILE" 223} 224 225write_sysroot_path() { 226 local -r package="$1" version="$2" 227 if [[ "$package" = sysroot ]]; then 228 echo "SYSROOT_${PLATFORM}_PATH = \$(LKMAKEROOT)/prebuilt/downloads/sysroot" 229 fi 230} 231 232write_config_mk() { 233 local -r internal_access="$1" 234 local -r config_mk="${PREBUILTS_DIR}/config.mk" 235 rm -f -- "$config_mk" 236 echo > "$config_mk" "# Generated by $0. DO NOT EDIT!"' 237 238PREBUILT_CHECK := $(shell $(LKMAKEROOT)/scripts/download-prebuilt --verify) 239ifneq (,$(strip $(PREBUILT_CHECK))) 240$(warning) 241$(warning $(PREBUILT_CHECK)) 242$(warning run scripts/download-prebuilt) 243$(warning) 244endif 245 246ARCH_x86_64_TOOLCHAIN_PREFIX = $(LKMAKEROOT)/prebuilt/downloads/gcc/bin/x86_64-elf- 247ARCH_arm64_TOOLCHAIN_PREFIX = $(LKMAKEROOT)/prebuilt/downloads/gcc/bin/aarch64-elf- 248CLANG_TOOLCHAIN_PREFIX = $(LKMAKEROOT)/prebuilt/downloads/clang/bin/' 249 250 for_each_package write_sysroot_path < "$VERSIONS_FILE" >> "$config_mk" 251 252 echo >> "$config_mk" "INTERNAL_ACCESS := ${internal_access}" 253} 254 255update() { 256 local CIPD 257 local internal_access=false 258 if $cipd_ok && CIPD="$(find_cipd)"; then 259 # We have CIPD, so use it. 260 if [[ "$("$CIPD" ls fuchsia_internal)" != "No matching packages." ]]; then 261 internal_access=true 262 fi 263 update_via_cipd "$CIPD" "$internal_access" || return 264 else 265 # We don't have CIPD, so don't use it. 266 for_each_package download_package < "$VERSIONS_FILE" || return 267 fi 268 write_config_mk "$internal_access" 269} 270 271verify() { 272 for_each_package verify_stamp --verbose < "$VERSIONS_FILE" 273} 274 275list() { 276 for_each_package list_package < "$VERSIONS_FILE" 277} 278 279$mode 280