1# -*- coding: utf-8 -*- 2# 3# Copyright 2015 Michael Lotz 4# Copyright 2016 Jerome Duval 5# Distributed under the terms of the MIT License. 6 7import errno 8import json 9import logging 10import os 11import socket 12import stat 13import time 14 15# These usages kinda need refactored 16from ..ConfigParser import ConfigParser 17from ..Configuration import Configuration 18from .Builder import BuilderState 19 20try: 21 import paramiko 22except ImportError: 23 paramiko = None 24 25class RemoteBuilderSSH(object): 26 def __init__(self, configFilePath, packagesPath, outputBaseDir, 27 portsTreeOriginURL, portsTreeHead): 28 self._loadConfig(configFilePath) 29 self.availablePackages = [] 30 self.visiblePackages = [] 31 self.portsTreeOriginURL = portsTreeOriginURL 32 self.portsTreeHead = portsTreeHead 33 self.packagesPath = packagesPath 34 35 if not paramiko: 36 raise Exception('paramiko unavailable') 37 38 self.builderOutputDir = os.path.join(outputBaseDir, 'builders') 39 if not os.path.isdir(self.builderOutputDir): 40 os.makedirs(self.builderOutputDir) 41 42 self.buildOutputDir = os.path.join(outputBaseDir, 'builds') 43 if not os.path.isdir(self.buildOutputDir): 44 os.makedirs(self.buildOutputDir) 45 46 self.state = BuilderState.NOT_AVAILABLE 47 self.connectionErrors = 0 48 self.maxConnectionErrors = 100 49 50 self.currentBuild = None 51 52 self.logger = logging.getLogger('builders.' + self.name) 53 self.logger.setLevel(logging.DEBUG) 54 55 if 'hostKeyFile' not in self.config['ssh']: 56 self.logger.warning('Missing hostKeyFile for builder ' + self.name) 57 58 formatter = logging.Formatter('%(asctime)s: %(message)s') 59 logHandler = logging.FileHandler( 60 os.path.join(self.builderOutputDir, self.name + '.log'), 61 encoding='utf-8') 62 logHandler.setFormatter(formatter) 63 self.logger.addHandler(logHandler) 64 65 self.buildLogger = logging.getLogger('builders.' + self.name + '.build') 66 self.buildLogger.setLevel(logging.DEBUG) 67 68 def _loadConfig(self, configFilePath): 69 with open(configFilePath, 'r') as configFile: 70 self.config = json.loads(configFile.read()) 71 72 if 'name' not in self.config: 73 raise Exception('missing name in ' + configFilePath) 74 75 self.name = self.config['name'] 76 77 if 'ssh' not in self.config: 78 raise Exception('missing ssh config for builder ' + self.name) 79 if 'port' not in self.config['ssh']: 80 self.config['ssh']['port'] = 22 81 if 'user' not in self.config['ssh']: 82 raise Exception('missing ssh user config for builder ' + self.name) 83 if 'host' not in self.config['ssh']: 84 raise Exception('missing ssh host config for builder ' + self.name) 85 if 'privateKeyFile' not in self.config['ssh']: 86 raise Exception('missing ssh privateKeyFile config for builder ' 87 + self.name) 88 if not os.path.isabs(self.config['ssh']['privateKeyFile']): 89 self.config['ssh']['privateKeyFile'] = os.path.join( 90 os.path.dirname(configFilePath), 91 self.config['ssh']['privateKeyFile']) 92 93 if 'hostKeyFile' not in self.config['ssh']: 94 raise Exception('missing ssh hostKeyFile config for builder' + self.name) 95 if not os.path.isabs(self.config['ssh']['hostKeyFile']): 96 self.config['ssh']['hostKeyFile'] = os.path.join( 97 os.path.dirname(configFilePath), 98 self.config['ssh']['hostKeyFile']) 99 100 if 'portstree' not in self.config: 101 raise Exception('missing portstree config for builder ' + self.name) 102 if 'path' not in self.config['portstree']: 103 raise Exception('missing portstree path config for builder ' 104 + self.name) 105 if 'packagesPath' not in self.config['portstree']: 106 self.config['portstree']['packagesPath'] \ 107 = self.config['portstree']['path'] + '/packages' 108 if 'packagesCachePath' not in self.config['portstree']: 109 self.config['portstree']['packagesCachePath'] \ 110 = self.config['portstree']['packagesPath'] + '/.cache' 111 if 'builderConfig' not in self.config['portstree']: 112 self.config['portstree']['builderConfig'] \ 113 = self.config['portstree']['path'] + '/builder.conf' 114 115 if 'haikuporter' not in self.config: 116 self.config['haikuporter'] = {} 117 if 'path' not in self.config['haikuporter']: 118 self.config['haikuporter']['path'] = 'haikuporter' 119 if 'args' not in self.config['haikuporter']: 120 self.config['haikuporter']['args'] = '' 121 122 def _connect(self): 123 try: 124 self.sshClient = paramiko.SSHClient() 125 self.sshClient.load_host_keys(self.config['ssh']['hostKeyFile']) 126 self.logger.info('trying to connect to builder ' + self.name) 127 self.sshClient.connect(hostname=self.config['ssh']['host'], 128 port=int(self.config['ssh']['port']), 129 username=self.config['ssh']['user'], 130 key_filename=self.config['ssh']['privateKeyFile'], 131 compress=True, allow_agent=False, look_for_keys=False, 132 timeout=10) 133 134 self.sshClient.get_transport().set_keepalive(15) 135 self.sftpClient = self.sshClient.open_sftp() 136 137 self.logger.info('connected to builder') 138 self.connectionErrors = 0 139 except Exception as exception: 140 self.logger.error('failed to connect to builder: ' 141 + str(exception)) 142 143 self.connectionErrors += 1 144 self.state = BuilderState.RECONNECT 145 146 if self.connectionErrors >= self.maxConnectionErrors: 147 self.logger.error('giving up on builder after ' 148 + str(self.connectionErrors) 149 + ' consecutive connection errors') 150 self.state = BuilderState.LOST 151 raise 152 153 # avoid DoSing the remote host, increasing delay as retries increase. 154 time.sleep(5 + (1.2 * self.connectionErrors)) 155 raise 156 157 def _validatePortsTree(self): 158 try: 159 command = ('if [ ! -d "' + self.config['portstree']['path'] + '" ]; ' 160 + 'then git clone "' + self.portsTreeOriginURL + '" ' 161 + self.config['portstree']['path'] + '; fi') 162 self.logger.info('running command: ' + command) 163 (output, channel) = self._remoteCommand(command) 164 return channel.recv_exit_status() == 0 165 except Exception as exception: 166 self.logger.error('failed to validate ports tree: ' 167 + str(exception)) 168 raise 169 170 def _syncPortsTree(self): 171 try: 172 command = ('cd "' + self.config['portstree']['path'] 173 + '" && git fetch && git checkout ' + self.portsTreeHead) 174 self.logger.info('running command: ' + command) 175 (output, channel) = self._remoteCommand(command) 176 if channel.recv_exit_status() != 0: 177 raise Exception('sync command failed') 178 except Exception as exception: 179 self.logger.error('failed to sync ports tree: ' 180 + str(exception)) 181 raise 182 183 def _writeBuilderConfig(self): 184 try: 185 config = { 186 'TREE_PATH': self.config['portstree']['path'], 187 'PACKAGES_PATH': self.config['portstree']['packagesPath'], 188 'PACKAGER': 'Builder ' + self.name \ 189 + ' <hpkg-builder@haiku-os.org>', 190 'TARGET_ARCHITECTURE': Configuration.getTargetArchitecture(), 191 'SECONDARY_TARGET_ARCHITECTURES': \ 192 Configuration.getSecondaryTargetArchitectures(), 193 'ALLOW_UNTESTED': Configuration.shallAllowUntested(), 194 'ALLOW_UNSAFE_SOURCES': Configuration.shallAllowUnsafeSources(), 195 'CREATE_SOURCE_PACKAGES': Configuration.shallCreateSourcePackages() 196 } 197 198 with self._openRemoteFile(self.config['portstree']['builderConfig'], 199 'w') as remoteFile: 200 remoteFile.write( 201 ConfigParser.configurationStringFromDict(config)) 202 except Exception as exception: 203 self.logger.error('failed to write builder config: ' 204 + str(exception)) 205 raise 206 207 def _createNeededDirs(self): 208 try: 209 self._ensureDirExists(self.config['portstree']['packagesPath']) 210 self._ensureDirExists(self.config['portstree']['packagesCachePath']) 211 except Exception as exception: 212 self.logger.error('failed to create needed dirs: ' 213 + str(exception)) 214 raise 215 216 def _getAvailablePackages(self): 217 try: 218 self._clearVisiblePackages() 219 220 for entry in self._listDir( 221 self.config['portstree']['packagesCachePath']): 222 if not entry.endswith('.hpkg'): 223 continue 224 225 if entry not in self.availablePackages: 226 self.availablePackages.append(entry) 227 except Exception as exception: 228 self.logger.error('failed to get available packages: ' 229 + str(exception)) 230 raise 231 232 def _removeObsoletePackages(self): 233 cachePath = self.config['portstree']['packagesCachePath'] 234 for entry in list(self.availablePackages): 235 if not os.path.exists(os.path.join(self.packagesPath, entry)): 236 self.logger.info( 237 'removing obsolete package {} from cache'.format(entry)) 238 entryPath = cachePath + '/' + entry 239 self.sftpClient.remove(entryPath) 240 self.availablePackages.remove(entry) 241 242 def _setupForBuilding(self): 243 if self.state == BuilderState.AVAILABLE: 244 return True 245 if self.state == BuilderState.LOST: 246 return False 247 248 self._connect() 249 self._validatePortsTree() 250 self._syncPortsTree() 251 self._writeBuilderConfig() 252 self._createNeededDirs() 253 self._getAvailablePackages() 254 self._removeObsoletePackages() 255 256 self.state = BuilderState.AVAILABLE 257 return True 258 259 def setBuild(self, scheduledBuild, buildNumber): 260 logHandler = logging.FileHandler(os.path.join(self.buildOutputDir, 261 str(buildNumber) + '.log'), encoding='utf-8') 262 logHandler.setFormatter(logging.Formatter('%(message)s')) 263 self.buildLogger.addHandler(logHandler) 264 265 self.currentBuild = { 266 'build': scheduledBuild, 267 'status': scheduledBuild.status, 268 'number': buildNumber, 269 'logHandler': logHandler 270 } 271 272 def unsetBuild(self): 273 self.buildLogger.removeHandler(self.currentBuild['logHandler']) 274 self.currentBuild = None 275 276 def runBuild(self): 277 scheduledBuild = self.currentBuild['build'] 278 buildSuccess = False 279 reschedule = True 280 281 try: 282 if not self._setupForBuilding(): 283 return (False, True) 284 285 self._purgePort(scheduledBuild) 286 self._clearVisiblePackages() 287 for requiredPackage in scheduledBuild.requiredPackages: 288 self._makePackageAvailable(requiredPackage) 289 self._makePackageVisible(requiredPackage) 290 291 self.buildLogger.info('building port ' 292 + scheduledBuild.port.versionedName) 293 294 # TODO: We don't actually want to source the build host environment 295 # but the one from within the provided Haiku package. This does 296 # clash with the manipulation of PATH that is done by haikuporter 297 # to support secondary architectures and cross builds. Ideally the 298 # shell scriptlet to set up the chroot environment would take over 299 # these tasks and would initially source the environment from within 300 # the chroot and then do any necessary manipulation. 301 command = ('source /boot/system/boot/SetupEnvironment' 302 + ' && cd "' + self.config['portstree']['path'] 303 + '" && "' + self.config['haikuporter']['path'] 304 + '" --config="' + self.config['portstree']['builderConfig'] 305 + '" --no-system-packages --no-package-obsoletion' 306 + ' --ignore-messages ' 307 + self.config['haikuporter']['args'] + ' "' 308 + scheduledBuild.port.versionedName + '"') 309 310 self.buildLogger.info('running command: ' + command) 311 self.buildLogger.propagate = False 312 313 (output, channel) = self._remoteCommand(command) 314 self._appendOutputToLog(output) 315 316 self.buildLogger.propagate = True 317 exitStatus = channel.recv_exit_status() 318 self.buildLogger.info('command exit status: ' + str(exitStatus)) 319 320 if exitStatus < 0 and not channel.get_transport().is_active(): 321 self.state = BuilderState.NOT_AVAILABLE 322 raise Exception('builder disconnected') 323 324 if exitStatus != 0: 325 reschedule = False 326 self._purgePort(scheduledBuild) 327 self._clearVisiblePackages() 328 raise Exception('build failure') 329 330 for package in scheduledBuild.port.packages: 331 self.buildLogger.info('download package ' + package.hpkgName 332 + ' from builder') 333 334 packageFile = os.path.join(self.packagesPath, package.hpkgName) 335 downloadFile = packageFile + '.download' 336 self._getFile(self.config['portstree']['packagesPath'] + '/' 337 + package.hpkgName, downloadFile) 338 os.rename(downloadFile, packageFile) 339 340 self._purgePort(scheduledBuild) 341 self._clearVisiblePackages() 342 self.buildLogger.info('build completed successfully') 343 buildSuccess = True 344 345 except socket.error as exception: 346 self.buildLogger.error('connection failed: ' + str(exception)) 347 if self.state == BuilderState.AVAILABLE: 348 self.state = BuilderState.RECONNECT 349 350 except (IOError, paramiko.ssh_exception.SSHException) as exception: 351 self.buildLogger.error('builder failed: ' + str(exception)) 352 self.state = BuilderState.LOST 353 354 except Exception as exception: 355 self.buildLogger.info('build failed: ' + str(exception)) 356 357 return (buildSuccess, reschedule) 358 359 def _remoteCommand(self, command): 360 transport = self.sshClient.get_transport() 361 channel = transport.open_session() 362 channel.get_pty() 363 output = channel.makefile('rb') 364 channel.exec_command(command) 365 return (output, channel) 366 367 def _getFile(self, localPath, remotePath): 368 self.sftpClient.get(localPath, remotePath) 369 370 def _putFile(self, remotePath, localPath): 371 self.sftpClient.put(remotePath, localPath) 372 373 def _symlink(self, sourcePath, destPath): 374 self.sftpClient.symlink(sourcePath, destPath) 375 376 def _move(self, sourcePath, destPath): 377 # Unfortunately we can't use SFTPClient.rename as that uses the rename 378 # command (vs. posix-rename) which uses hardlinks which fail on BFS 379 (output, channel) = self._remoteCommand('mv "' + sourcePath + '" "' 380 + destPath + '"') 381 if channel.recv_exit_status() != 0: 382 raise IOError('failed moving {} to {}'.format(sourcePath, destPath)) 383 384 def _openRemoteFile(self, path, mode): 385 return self.sftpClient.open(path, mode) 386 387 def _ensureDirExists(self, path): 388 try: 389 attributes = self.sftpClient.stat(path) 390 if not stat.S_ISDIR(attributes.st_mode): 391 raise IOError(errno.EEXIST, 'file exists') 392 except IOError as exception: 393 if exception.errno != errno.ENOENT: 394 raise 395 396 self.sftpClient.mkdir(path) 397 398 def _listDir(self, remotePath): 399 return self.sftpClient.listdir(remotePath) 400 401 def _purgePort(self, scheduledBuild): 402 command = ('cd "' + self.config['portstree']['path'] 403 + '" && "' + self.config['haikuporter']['path'] 404 + '" --config="' + self.config['portstree']['builderConfig'] 405 + '" --no-package-obsoletion --ignore-messages --purge "' 406 + scheduledBuild.port.versionedName + '"') 407 408 self.buildLogger.info('purging port with command: ' + command) 409 (output, channel) = self._remoteCommand(command) 410 self._appendOutputToLog(output) 411 412 def _appendOutputToLog(self, output): 413 with output: 414 while True: 415 line = output.readline() 416 if not line: 417 return 418 419 self.buildLogger.info( 420 line[:-1].decode('utf-8', errors='replace')) 421 422 def _makePackageAvailable(self, packagePath): 423 packageName = os.path.basename(packagePath) 424 if packageName in self.availablePackages: 425 return 426 427 self.logger.info('upload package ' + packageName + ' to builder') 428 429 entryPath \ 430 = self.config['portstree']['packagesCachePath'] + '/' + packageName 431 uploadPath = entryPath + '.upload' 432 433 self._putFile(packagePath, uploadPath) 434 self._move(uploadPath, entryPath) 435 436 self.availablePackages.append(packageName) 437 438 def _clearVisiblePackages(self): 439 basePath = self.config['portstree']['packagesPath'] 440 cachePath = self.config['portstree']['packagesCachePath'] 441 for entry in self._listDir(basePath): 442 if not entry.endswith('.hpkg'): 443 continue 444 445 entryPath = basePath + '/' + entry 446 attributes = self.sftpClient.lstat(entryPath) 447 if stat.S_ISLNK(attributes.st_mode): 448 self.logger.debug('removing symlink to package ' + entry) 449 self.sftpClient.remove(entryPath) 450 else: 451 self.logger.info('moving package ' + entry + ' to cache') 452 cacheEntryPath = cachePath + '/' + entry 453 self._move(entryPath, cacheEntryPath) 454 self.availablePackages.append(entry) 455 456 self.visiblePackages = [] 457 458 def _makePackageVisible(self, packagePath): 459 packageName = os.path.basename(packagePath) 460 if packageName in self.visiblePackages: 461 return 462 463 self.logger.debug('making package ' + packageName + ' visible') 464 self._symlink( 465 self.config['portstree']['packagesCachePath'] + '/' + packageName, 466 self.config['portstree']['packagesPath'] + '/' + packageName) 467 468 self.visiblePackages.append(packageName) 469 470 @property 471 def status(self): 472 return { 473 'name': self.name, 474 'state': self.state, 475 'availablePackages': self.availablePackages, 476 'connectionErrors': self.connectionErrors, 477 'maxConnectionErrors': self.maxConnectionErrors, 478 'currentBuild': { 479 'build': self.currentBuild['status'], 480 'number': self.currentBuild['number'] 481 } if self.currentBuild else None 482 } 483