1# -*- coding: utf-8 -*-
2#
3# Copyright 2015 Michael Lotz
4# Copyright 2016 Jerome Duval
5# Distributed under the terms of the MIT License.
6
7import errno
8import json
9import logging
10import os
11import socket
12import stat
13import time
14
15# These usages kinda need refactored
16from ..ConfigParser import ConfigParser
17from ..Configuration import Configuration
18from .Builder import BuilderState
19
20try:
21	import paramiko
22except ImportError:
23	paramiko = None
24
25class RemoteBuilderSSH(object):
26	def __init__(self, configFilePath, packagesPath, outputBaseDir,
27			portsTreeOriginURL, portsTreeHead):
28		self._loadConfig(configFilePath)
29		self.availablePackages = []
30		self.visiblePackages = []
31		self.portsTreeOriginURL = portsTreeOriginURL
32		self.portsTreeHead = portsTreeHead
33		self.packagesPath = packagesPath
34
35		if not paramiko:
36			raise Exception('paramiko unavailable')
37
38		self.builderOutputDir = os.path.join(outputBaseDir, 'builders')
39		if not os.path.isdir(self.builderOutputDir):
40			os.makedirs(self.builderOutputDir)
41
42		self.buildOutputDir = os.path.join(outputBaseDir, 'builds')
43		if not os.path.isdir(self.buildOutputDir):
44			os.makedirs(self.buildOutputDir)
45
46		self.state = BuilderState.NOT_AVAILABLE
47		self.connectionErrors = 0
48		self.maxConnectionErrors = 100
49
50		self.currentBuild = None
51
52		self.logger = logging.getLogger('builders.' + self.name)
53		self.logger.setLevel(logging.DEBUG)
54
55		if 'hostKeyFile' not in self.config['ssh']:
56			self.logger.warning('Missing hostKeyFile for builder ' + self.name)
57
58		formatter = logging.Formatter('%(asctime)s: %(message)s')
59		logHandler = logging.FileHandler(
60			os.path.join(self.builderOutputDir, self.name + '.log'),
61			encoding='utf-8')
62		logHandler.setFormatter(formatter)
63		self.logger.addHandler(logHandler)
64
65		self.buildLogger = logging.getLogger('builders.' + self.name + '.build')
66		self.buildLogger.setLevel(logging.DEBUG)
67
68	def _loadConfig(self, configFilePath):
69		with open(configFilePath, 'r') as configFile:
70			self.config = json.loads(configFile.read())
71
72		if 'name' not in self.config:
73			raise Exception('missing name in ' + configFilePath)
74
75		self.name = self.config['name']
76
77		if 'ssh' not in self.config:
78			raise Exception('missing ssh config for builder ' + self.name)
79		if 'port' not in self.config['ssh']:
80			self.config['ssh']['port'] = 22
81		if 'user' not in self.config['ssh']:
82			raise Exception('missing ssh user config for builder ' + self.name)
83		if 'host' not in self.config['ssh']:
84			raise Exception('missing ssh host config for builder ' + self.name)
85		if 'privateKeyFile' not in self.config['ssh']:
86			raise Exception('missing ssh privateKeyFile config for builder '
87				+ self.name)
88		if not os.path.isabs(self.config['ssh']['privateKeyFile']):
89			self.config['ssh']['privateKeyFile'] = os.path.join(
90				os.path.dirname(configFilePath),
91				self.config['ssh']['privateKeyFile'])
92
93		if 'hostKeyFile' not in self.config['ssh']:
94			raise Exception('missing ssh hostKeyFile config for builder' + self.name)
95		if not os.path.isabs(self.config['ssh']['hostKeyFile']):
96			self.config['ssh']['hostKeyFile'] = os.path.join(
97				os.path.dirname(configFilePath),
98				self.config['ssh']['hostKeyFile'])
99
100		if 'portstree' not in self.config:
101			raise Exception('missing portstree config for builder ' + self.name)
102		if 'path' not in self.config['portstree']:
103			raise Exception('missing portstree path config for builder '
104				+ self.name)
105		if 'packagesPath' not in self.config['portstree']:
106			self.config['portstree']['packagesPath'] \
107				= self.config['portstree']['path'] + '/packages'
108		if 'packagesCachePath' not in self.config['portstree']:
109			self.config['portstree']['packagesCachePath'] \
110				= self.config['portstree']['packagesPath'] + '/.cache'
111		if 'builderConfig' not in self.config['portstree']:
112			self.config['portstree']['builderConfig'] \
113				= self.config['portstree']['path'] + '/builder.conf'
114
115		if 'haikuporter' not in self.config:
116			self.config['haikuporter'] = {}
117		if 'path' not in self.config['haikuporter']:
118			self.config['haikuporter']['path'] = 'haikuporter'
119		if 'args' not in self.config['haikuporter']:
120			self.config['haikuporter']['args'] = ''
121
122	def _connect(self):
123		try:
124			self.sshClient = paramiko.SSHClient()
125			self.sshClient.load_host_keys(self.config['ssh']['hostKeyFile'])
126			self.logger.info('trying to connect to builder ' + self.name)
127			self.sshClient.connect(hostname=self.config['ssh']['host'],
128				port=int(self.config['ssh']['port']),
129				username=self.config['ssh']['user'],
130				key_filename=self.config['ssh']['privateKeyFile'],
131				compress=True, allow_agent=False, look_for_keys=False,
132				timeout=10)
133
134			self.sshClient.get_transport().set_keepalive(15)
135			self.sftpClient = self.sshClient.open_sftp()
136
137			self.logger.info('connected to builder')
138			self.connectionErrors = 0
139		except Exception as exception:
140			self.logger.error('failed to connect to builder: '
141				+ str(exception))
142
143			self.connectionErrors += 1
144			self.state = BuilderState.RECONNECT
145
146			if self.connectionErrors >= self.maxConnectionErrors:
147				self.logger.error('giving up on builder after '
148					+ str(self.connectionErrors)
149					+ ' consecutive connection errors')
150				self.state = BuilderState.LOST
151				raise
152
153			# avoid DoSing the remote host, increasing delay as retries increase.
154			time.sleep(5 + (1.2 * self.connectionErrors))
155			raise
156
157	def _validatePortsTree(self):
158		try:
159			command = ('if [ ! -d "' + self.config['portstree']['path'] + '" ]; '
160				+ 'then git clone "' + self.portsTreeOriginURL + '" '
161				+ self.config['portstree']['path'] + '; fi')
162			self.logger.info('running command: ' + command)
163			(output, channel) = self._remoteCommand(command)
164			return channel.recv_exit_status() == 0
165		except Exception as exception:
166			self.logger.error('failed to validate ports tree: '
167				+ str(exception))
168			raise
169
170	def _syncPortsTree(self):
171		try:
172			command = ('cd "' + self.config['portstree']['path']
173				+ '" && git fetch && git checkout ' + self.portsTreeHead)
174			self.logger.info('running command: ' + command)
175			(output, channel) = self._remoteCommand(command)
176			if channel.recv_exit_status() != 0:
177				raise Exception('sync command failed')
178		except Exception as exception:
179			self.logger.error('failed to sync ports tree: '
180				+ str(exception))
181			raise
182
183	def _writeBuilderConfig(self):
184		try:
185			config = {
186				'TREE_PATH': self.config['portstree']['path'],
187				'PACKAGES_PATH': self.config['portstree']['packagesPath'],
188				'PACKAGER': 'Builder ' + self.name \
189					+ ' <hpkg-builder@haiku-os.org>',
190				'TARGET_ARCHITECTURE': Configuration.getTargetArchitecture(),
191				'SECONDARY_TARGET_ARCHITECTURES': \
192					Configuration.getSecondaryTargetArchitectures(),
193				'ALLOW_UNTESTED': Configuration.shallAllowUntested(),
194				'ALLOW_UNSAFE_SOURCES': Configuration.shallAllowUnsafeSources(),
195				'CREATE_SOURCE_PACKAGES': Configuration.shallCreateSourcePackages()
196			}
197
198			with self._openRemoteFile(self.config['portstree']['builderConfig'],
199					'w') as remoteFile:
200				remoteFile.write(
201					ConfigParser.configurationStringFromDict(config))
202		except Exception as exception:
203			self.logger.error('failed to write builder config: '
204				+ str(exception))
205			raise
206
207	def _createNeededDirs(self):
208		try:
209			self._ensureDirExists(self.config['portstree']['packagesPath'])
210			self._ensureDirExists(self.config['portstree']['packagesCachePath'])
211		except Exception as exception:
212			self.logger.error('failed to create needed dirs: '
213				+ str(exception))
214			raise
215
216	def _getAvailablePackages(self):
217		try:
218			self._clearVisiblePackages()
219
220			for entry in self._listDir(
221					self.config['portstree']['packagesCachePath']):
222				if not entry.endswith('.hpkg'):
223					continue
224
225				if entry not in self.availablePackages:
226					self.availablePackages.append(entry)
227		except Exception as exception:
228			self.logger.error('failed to get available packages: '
229				+ str(exception))
230			raise
231
232	def _removeObsoletePackages(self):
233		cachePath = self.config['portstree']['packagesCachePath']
234		for entry in list(self.availablePackages):
235			if not os.path.exists(os.path.join(self.packagesPath, entry)):
236				self.logger.info(
237					'removing obsolete package {} from cache'.format(entry))
238				entryPath = cachePath + '/' + entry
239				self.sftpClient.remove(entryPath)
240				self.availablePackages.remove(entry)
241
242	def _setupForBuilding(self):
243		if self.state == BuilderState.AVAILABLE:
244			return True
245		if self.state == BuilderState.LOST:
246			return False
247
248		self._connect()
249		self._validatePortsTree()
250		self._syncPortsTree()
251		self._writeBuilderConfig()
252		self._createNeededDirs()
253		self._getAvailablePackages()
254		self._removeObsoletePackages()
255
256		self.state = BuilderState.AVAILABLE
257		return True
258
259	def setBuild(self, scheduledBuild, buildNumber):
260		logHandler = logging.FileHandler(os.path.join(self.buildOutputDir,
261				str(buildNumber) + '.log'), encoding='utf-8')
262		logHandler.setFormatter(logging.Formatter('%(message)s'))
263		self.buildLogger.addHandler(logHandler)
264
265		self.currentBuild = {
266			'build': scheduledBuild,
267			'status': scheduledBuild.status,
268			'number': buildNumber,
269			'logHandler': logHandler
270		}
271
272	def unsetBuild(self):
273		self.buildLogger.removeHandler(self.currentBuild['logHandler'])
274		self.currentBuild = None
275
276	def runBuild(self):
277		scheduledBuild = self.currentBuild['build']
278		buildSuccess = False
279		reschedule = True
280
281		try:
282			if not self._setupForBuilding():
283				return (False, True)
284
285			self._purgePort(scheduledBuild)
286			self._clearVisiblePackages()
287			for requiredPackage in scheduledBuild.requiredPackages:
288				self._makePackageAvailable(requiredPackage)
289				self._makePackageVisible(requiredPackage)
290
291			self.buildLogger.info('building port '
292				+ scheduledBuild.port.versionedName)
293
294			# TODO: We don't actually want to source the build host environment
295			# but the one from within the provided Haiku package. This does
296			# clash with the manipulation of PATH that is done by haikuporter
297			# to support secondary architectures and cross builds. Ideally the
298			# shell scriptlet to set up the chroot environment would take over
299			# these tasks and would initially source the environment from within
300			# the chroot and then do any necessary manipulation.
301			command = ('source /boot/system/boot/SetupEnvironment'
302				+ ' && cd "' + self.config['portstree']['path']
303				+ '" && "' + self.config['haikuporter']['path']
304				+ '" --config="' + self.config['portstree']['builderConfig']
305				+ '" --no-system-packages --no-package-obsoletion'
306				+ ' --ignore-messages '
307				+ self.config['haikuporter']['args'] + ' "'
308				+ scheduledBuild.port.versionedName + '"')
309
310			self.buildLogger.info('running command: ' + command)
311			self.buildLogger.propagate = False
312
313			(output, channel) = self._remoteCommand(command)
314			self._appendOutputToLog(output)
315
316			self.buildLogger.propagate = True
317			exitStatus = channel.recv_exit_status()
318			self.buildLogger.info('command exit status: ' + str(exitStatus))
319
320			if exitStatus < 0 and not channel.get_transport().is_active():
321				self.state = BuilderState.NOT_AVAILABLE
322				raise Exception('builder disconnected')
323
324			if exitStatus != 0:
325				reschedule = False
326				self._purgePort(scheduledBuild)
327				self._clearVisiblePackages()
328				raise Exception('build failure')
329
330			for package in scheduledBuild.port.packages:
331				self.buildLogger.info('download package ' + package.hpkgName
332					+ ' from builder')
333
334				packageFile = os.path.join(self.packagesPath, package.hpkgName)
335				downloadFile = packageFile + '.download'
336				self._getFile(self.config['portstree']['packagesPath'] + '/'
337						+ package.hpkgName, downloadFile)
338				os.rename(downloadFile, packageFile)
339
340			self._purgePort(scheduledBuild)
341			self._clearVisiblePackages()
342			self.buildLogger.info('build completed successfully')
343			buildSuccess = True
344
345		except socket.error as exception:
346			self.buildLogger.error('connection failed: ' + str(exception))
347			if self.state == BuilderState.AVAILABLE:
348				self.state = BuilderState.RECONNECT
349
350		except (IOError, paramiko.ssh_exception.SSHException) as exception:
351			self.buildLogger.error('builder failed: ' + str(exception))
352			self.state = BuilderState.LOST
353
354		except Exception as exception:
355			self.buildLogger.info('build failed: ' + str(exception))
356
357		return (buildSuccess, reschedule)
358
359	def _remoteCommand(self, command):
360		transport = self.sshClient.get_transport()
361		channel = transport.open_session()
362		channel.get_pty()
363		output = channel.makefile('rb')
364		channel.exec_command(command)
365		return (output, channel)
366
367	def _getFile(self, localPath, remotePath):
368		self.sftpClient.get(localPath, remotePath)
369
370	def _putFile(self, remotePath, localPath):
371		self.sftpClient.put(remotePath, localPath)
372
373	def _symlink(self, sourcePath, destPath):
374		self.sftpClient.symlink(sourcePath, destPath)
375
376	def _move(self, sourcePath, destPath):
377		# Unfortunately we can't use SFTPClient.rename as that uses the rename
378		# command (vs. posix-rename) which uses hardlinks which fail on BFS
379		(output, channel) = self._remoteCommand('mv "' + sourcePath + '" "'
380			+ destPath + '"')
381		if channel.recv_exit_status() != 0:
382			raise IOError('failed moving {} to {}'.format(sourcePath, destPath))
383
384	def _openRemoteFile(self, path, mode):
385		return self.sftpClient.open(path, mode)
386
387	def _ensureDirExists(self, path):
388		try:
389			attributes = self.sftpClient.stat(path)
390			if not stat.S_ISDIR(attributes.st_mode):
391				raise IOError(errno.EEXIST, 'file exists')
392		except IOError as exception:
393			if exception.errno != errno.ENOENT:
394				raise
395
396			self.sftpClient.mkdir(path)
397
398	def _listDir(self, remotePath):
399		return self.sftpClient.listdir(remotePath)
400
401	def _purgePort(self, scheduledBuild):
402		command = ('cd "' + self.config['portstree']['path']
403			+ '" && "' + self.config['haikuporter']['path']
404			+ '" --config="' + self.config['portstree']['builderConfig']
405			+ '" --no-package-obsoletion --ignore-messages --purge "'
406			+ scheduledBuild.port.versionedName + '"')
407
408		self.buildLogger.info('purging port with command: ' + command)
409		(output, channel) = self._remoteCommand(command)
410		self._appendOutputToLog(output)
411
412	def _appendOutputToLog(self, output):
413		with output:
414			while True:
415				line = output.readline()
416				if not line:
417					return
418
419				self.buildLogger.info(
420					line[:-1].decode('utf-8', errors='replace'))
421
422	def _makePackageAvailable(self, packagePath):
423		packageName = os.path.basename(packagePath)
424		if packageName in self.availablePackages:
425			return
426
427		self.logger.info('upload package ' + packageName + ' to builder')
428
429		entryPath \
430			= self.config['portstree']['packagesCachePath'] + '/' + packageName
431		uploadPath = entryPath + '.upload'
432
433		self._putFile(packagePath, uploadPath)
434		self._move(uploadPath, entryPath)
435
436		self.availablePackages.append(packageName)
437
438	def _clearVisiblePackages(self):
439		basePath = self.config['portstree']['packagesPath']
440		cachePath = self.config['portstree']['packagesCachePath']
441		for entry in self._listDir(basePath):
442			if not entry.endswith('.hpkg'):
443				continue
444
445			entryPath = basePath + '/' + entry
446			attributes = self.sftpClient.lstat(entryPath)
447			if stat.S_ISLNK(attributes.st_mode):
448				self.logger.debug('removing symlink to package ' + entry)
449				self.sftpClient.remove(entryPath)
450			else:
451				self.logger.info('moving package ' + entry + ' to cache')
452				cacheEntryPath = cachePath + '/' + entry
453				self._move(entryPath, cacheEntryPath)
454				self.availablePackages.append(entry)
455
456		self.visiblePackages = []
457
458	def _makePackageVisible(self, packagePath):
459		packageName = os.path.basename(packagePath)
460		if packageName in self.visiblePackages:
461			return
462
463		self.logger.debug('making package ' + packageName + ' visible')
464		self._symlink(
465			self.config['portstree']['packagesCachePath'] + '/' + packageName,
466			self.config['portstree']['packagesPath'] + '/' + packageName)
467
468		self.visiblePackages.append(packageName)
469
470	@property
471	def status(self):
472		return {
473			'name': self.name,
474			'state': self.state,
475			'availablePackages': self.availablePackages,
476			'connectionErrors': self.connectionErrors,
477			'maxConnectionErrors': self.maxConnectionErrors,
478			'currentBuild': {
479				'build': self.currentBuild['status'],
480				'number': self.currentBuild['number']
481			} if self.currentBuild else None
482		}
483