1# -*- coding: utf-8 -*- 2# 3# Copyright 2007-2011 Brecht Machiels 4# Copyright 2009-2010 Chris Roberts 5# Copyright 2009-2011 Scott McCreary 6# Copyright 2009 Alexander Deynichenko 7# Copyright 2009 HaikuBot (aka RISC) 8# Copyright 2010-2011 Jack Laxson (Jrabbit) 9# Copyright 2011 Ingo Weinhold 10# Copyright 2013 Oliver Tappe 11# Distributed under the terms of the MIT License. 12 13# -- Modules ------------------------------------------------------------------ 14 15import os 16import re 17import shutil 18import time 19from subprocess import PIPE, STDOUT, CalledProcessError, Popen, check_output 20 21from .Configuration import Configuration 22from .Utils import ensureCommandIsAvailable, info, sysExit, unpackArchive, warn 23 24# ----------------------------------------------------------------------------- 25 26def parseCheckoutUri(uri): 27 """Parse the given checkout URI and return a 3-tuple with type, real URI 28 and revision.""" 29 30 # Attempt to parse a URI with a + in it. ex: hg+http://blah 31 # If it doesn't find the 'type' it should extract 'real_uri' and 'rev' 32 m = re.match(r'^((?P<type>\w*)\+)?(?P<realUri>.+?)(#(?P<rev>.+))?$', uri) 33 if not m or not m.group('realUri'): 34 sysExit("Couldn't parse repository URI " + uri) 35 36 uriType = m.group('type') 37 realUri = m.group('realUri') 38 rev = m.group('rev') 39 40 # Attempt to parse a URI without a + in it. ex: svn://blah 41 if not uriType: 42 m = re.match(r'^(\w*).*$', realUri) 43 if m: 44 uriType = m.group(1) 45 46 if not uriType: 47 sysExit("Couldn't parse repository type from URI " + realUri) 48 49 return (uriType, realUri, rev) 50 51# ----------------------------------------------------------------------------- 52 53def unpackCheckoutWithTar(checkoutDir, sourceBaseDir, sourceSubDir, foldSubDir): 54 """Use 'tar' to export the sources from the checkout into the source dir""" 55 56 sourceDir = sourceBaseDir + '/' + sourceSubDir \ 57 if sourceSubDir else sourceBaseDir 58 if foldSubDir: 59 command = ('tar -c -C "%s" --exclude-vcs | tar -x -C "%s"' 60 % (foldSubDir, sourceDir)) 61 else: 62 command = 'tar -c --exclude-vcs . | tar -x -C "%s"' % sourceDir 63 output = check_output(command, cwd=checkoutDir, shell=True).decode('utf-8') 64 info(output) 65 66 if foldSubDir: 67 foldSubdirIntoSourceDir(foldSubDir, sourceDir) 68 69# ----------------------------------------------------------------------------- 70 71def unpackFile(uri, fetchTarget, sourceBaseDir, sourceSubDir, foldSubDir): 72 """Unpack archive file (or copy non-archive) into sourceDir""" 73 74 sourceDir = sourceBaseDir + '/' + sourceSubDir \ 75 if sourceSubDir else sourceBaseDir 76 if uri.endswith('#noarchive'): 77 if os.path.isdir(fetchTarget): 78 shutil.copytree(fetchTarget, sourceDir, symlinks=True) 79 else: 80 if not os.path.isdir(sourceDir): 81 os.makedirs(sourceDir) 82 shutil.copy(fetchTarget, sourceDir) 83 else: 84 actualSubDir = sourceSubDir 85 if actualSubDir: 86 if foldSubDir: 87 actualSubDir += '/' + foldSubDir 88 else: 89 actualSubDir = foldSubDir 90 unpackArchive(fetchTarget, sourceBaseDir, actualSubDir) 91 if foldSubDir: 92 foldSubdirIntoSourceDir(foldSubDir, sourceDir) 93 94# ----------------------------------------------------------------------------- 95 96def foldSubdirIntoSourceDir(subdir, sourceDir): 97 """Move contents of subdir into sourceDir and remove subdir""" 98 99 # rename subdir to something unique in order to avoid potential problems 100 # if it contains an identically named file or folder. 101 fullSubdirPath = sourceDir + '/subdir-to-be-folded-by-haikuporter' 102 os.rename(sourceDir + '/' + subdir, fullSubdirPath) 103 # now move all contents from the subdir into the source directory 104 for fileName in os.listdir(fullSubdirPath): 105 os.rename(fullSubdirPath + '/' + fileName, sourceDir + '/' + fileName) 106 os.removedirs(fullSubdirPath) 107 108# -- Fetches sources via bzr -------------------------------------------------- 109 110class SourceFetcherForBazaar(object): 111 def __init__(self, uri, fetchTarget): 112 self.fetchTarget = fetchTarget 113 self.sourceShouldBeValidated = False 114 115 (unusedType, self.uri, self.rev) = parseCheckoutUri(uri) 116 117 def fetch(self): 118 if not Configuration.shallAllowUnsafeSources(): 119 sysExit('Downloading from unsafe sources is disabled in ' + 120 'haikuports.conf!') 121 122 warn("UNSAFE SOURCES ARE BAD AND SHOULD NOT BE USED IN PRODUCTION") 123 warn("PLEASE MOVE TO A STATIC ARCHIVE DOWNLOAD WITH CHECKSUM ASAP!") 124 125 ensureCommandIsAvailable('bzr') 126 command = 'bzr checkout --lightweight' 127 if self.rev: 128 command += ' -r ' + self.rev 129 command += ' ' + self.uri + ' ' + self.fetchTarget 130 output = check_output(command, shell=True, stderr=STDOUT).decode('utf-8') 131 info(output) 132 133 def updateToRev(self, rev): 134 warn("Updating of a Bazaar repository to a specific revision has " 135 u"not been implemented yet, sorry") 136 137 def unpack(self, sourceBaseDir, sourceSubDir, foldSubDir): 138 unpackCheckoutWithTar(self.fetchTarget, sourceBaseDir, sourceSubDir, 139 foldSubDir) 140 141# -- Fetches sources via cvs -------------------------------------------------- 142 143class SourceFetcherForCvs(object): 144 def __init__(self, uri, fetchTarget): 145 self.fetchTarget = fetchTarget 146 self.sourceShouldBeValidated = False 147 148 (unusedType, uri, self.rev) = parseCheckoutUri(uri) 149 150 # chop the leading 'cvs://' of the URI, then split off the module 151 (self.uri, self.module) = uri[6:].rsplit('/', 1) 152 153 def fetch(self): 154 if not Configuration.shallAllowUnsafeSources(): 155 sysExit('Downloading from unsafe sources is disabled in ' + 156 'haikuports.conf!') 157 158 warn("UNSAFE SOURCES ARE BAD AND SHOULD NOT BE USED IN PRODUCTION") 159 warn("PLEASE MOVE TO A STATIC ARCHIVE DOWNLOAD WITH CHECKSUM ASAP!") 160 161 baseDir = os.path.dirname(self.fetchTarget) 162 163 ensureCommandIsAvailable('cvs') 164 command = 'cvs -d' + self.uri + ' co -P' 165 if self.rev: 166 # self.rev may specify a date or a revision/tag name. If it 167 # looks like a date, we assume it is one. 168 dateRegExp = re.compile(r'^\d{1,2}/\d{1,2}/\d{2,4}$|^\d{4}-\d{2}-\d{2}$') 169 if dateRegExp.match(self.rev): 170 command += ' -D' + self.rev 171 else: 172 command += ' -r' + self.rev 173 command += ' "%s"' % self.module 174 output = check_output(command, shell=True, cwd=baseDir, stderr=STDOUT).decode('utf-8') 175 info(output) 176 177 def updateToRev(self, rev): 178 warn("Updating of a CVS repository to a specific revision has " 179 u"not been implemented yet, sorry") 180 181 def unpack(self, sourceBaseDir, sourceSubDir, foldSubDir): 182 unpackCheckoutWithTar(self.fetchTarget, sourceBaseDir, sourceSubDir, 183 foldSubDir) 184 185# -- Fetches sources via wget ------------------------------------------------- 186 187class SourceFetcherForDownload(object): 188 def __init__(self, uri, fetchTarget): 189 self.fetchTarget = fetchTarget 190 self.uri = uri 191 self.sourceShouldBeValidated = True 192 193 def fetch(self): 194 downloadDir = os.path.dirname(self.fetchTarget) 195 ensureCommandIsAvailable('wget') 196 mirror = '' 197 if 'sourceforge.net/' in self.uri or '.sf.net/' in self.uri: 198 if Configuration.getSourceforgeMirror(): 199 mirror = '?use_mirror=' + Configuration.getSourceforgeMirror() 200 201 args = ['wget', '-c', '--tries=1', '--timeout=10', '--progress=dot:mega', '-O', 202 self.fetchTarget, self.uri + mirror] 203 204 code = 0 205 for tries in range(0, 3): 206 process = Popen(args, cwd=downloadDir, stdout=PIPE, stderr=STDOUT) 207 for line in iter(process.stdout.readline, b''): 208 info(line.decode('utf-8')[:-1]) 209 process.stdout.close() 210 code = process.wait() 211 if code in (0, 2, 6, 8): 212 # 0: success 213 # 2: parse error of command line 214 # 6: auth failure 215 # 8: error response from server 216 break 217 218 time.sleep(3) 219 220 if code: 221 raise CalledProcessError(code, args) 222 223 def updateToRev(self, rev): 224 pass 225 226 def unpack(self, sourceBaseDir, sourceSubDir, foldSubDir): 227 unpackFile(self.uri, self.fetchTarget, sourceBaseDir, sourceSubDir, 228 foldSubDir) 229 230# -- Fetches sources via fossil ----------------------------------------------- 231 232class SourceFetcherForFossil(object): 233 def __init__(self, uri, fetchTarget): 234 self.fetchTarget = fetchTarget 235 self.sourceShouldBeValidated = False 236 237 (unusedType, self.uri, self.rev) = parseCheckoutUri(uri) 238 239 def fetch(self): 240 if not Configuration.shallAllowUnsafeSources(): 241 sysExit('Downloading from unsafe sources is disabled in ' + 242 'haikuports.conf!') 243 244 warn("UNSAFE SOURCES ARE BAD AND SHOULD NOT BE USED IN PRODUCTION") 245 warn("PLEASE MOVE TO A STATIC ARCHIVE DOWNLOAD WITH CHECKSUM ASAP!") 246 247 ensureCommandIsAvailable('fossil') 248 fossilDir = self.fetchTarget + '.fossil' 249 if os.path.exists(fossilDir): 250 shutil.rmtree(fossilDir) 251 command = ('fossil clone ' + self.uri + ' ' + fossilDir 252 + ' && fossil open ' + fossilDir) 253 if self.rev: 254 command += ' ' + self.rev 255 output = check_output(command, shell=True, stderr=STDOUT).decode('utf-8') 256 info(output) 257 258 def updateToRev(self, rev): 259 warn("Updating of a Fossil repository to a specific revision has " 260 u"not been implemented yet, sorry") 261 262 def unpack(self, sourceBaseDir, sourceSubDir, foldSubDir): 263 unpackCheckoutWithTar(self.fetchTarget, sourceBaseDir, sourceSubDir, 264 foldSubDir) 265 266# -- Fetches sources via git -------------------------------------------------- 267 268class SourceFetcherForGit(object): 269 def __init__(self, uri, fetchTarget): 270 self.fetchTarget = fetchTarget 271 self.sourceShouldBeValidated = False 272 273 (unusedType, self.uri, self.rev) = parseCheckoutUri(uri) 274 if not self.rev: 275 self.rev = 'HEAD' 276 277 def fetch(self): 278 if not Configuration.shallAllowUnsafeSources(): 279 sysExit('Downloading from unsafe sources is disabled in ' + 280 'haikuports.conf!') 281 282 warn("UNSAFE SOURCES ARE BAD AND SHOULD NOT BE USED IN PRODUCTION") 283 warn("PLEASE MOVE TO A STATIC ARCHIVE DOWNLOAD WITH CHECKSUM ASAP!") 284 285 ensureCommandIsAvailable('git') 286 command = 'git clone --bare %s %s' % (self.uri, self.fetchTarget) 287 output = check_output(command, shell=True, stderr=STDOUT).decode('utf-8') 288 info(output) 289 290 def updateToRev(self, rev): 291 ensureCommandIsAvailable('git') 292 293 self.rev = rev 294 command = 'git rev-list --max-count=1 %s &>/dev/null' % self.rev 295 try: 296 output = check_output(command, shell=True, cwd=self.fetchTarget).decode('utf-8') 297 info(output) 298 except: 299 print('trying to fetch revision %s from upstream' % self.rev) 300 command = "git branch | cut -c3-" 301 branches = check_output(command, shell=True, 302 cwd=self.fetchTarget, stderr=STDOUT).decode('utf-8').splitlines() 303 for branch in branches: 304 command = 'git fetch origin %s:%s' % (branch, branch) 305 print(command) 306 output = check_output(command, shell=True, cwd=self.fetchTarget).decode('utf-8') 307 info(output) 308 # ensure that the revision really is available now 309 command = 'git rev-list --max-count=1 %s &>/dev/null' % self.rev 310 output = check_output(command, shell=True, cwd=self.fetchTarget).decode('utf-8') 311 info(output) 312 313 def unpack(self, sourceBaseDir, sourceSubDir, foldSubDir): 314 sourceDir = sourceBaseDir + '/' + sourceSubDir \ 315 if sourceSubDir else sourceBaseDir 316 if foldSubDir: 317 command = ('mkdir -p "%s" && git archive %s "%s" | tar -x -C "%s"' 318 % (sourceDir, self.rev, foldSubDir, sourceDir)) 319 else: 320 command = 'mkdir -p "%s" && git archive %s | tar -x -C "%s"' % (sourceDir, self.rev, sourceDir) 321 output = check_output(command, shell=True, cwd=self.fetchTarget).decode('utf-8') 322 info(output) 323 324 if foldSubDir: 325 foldSubdirIntoSourceDir(foldSubDir, sourceDir) 326 327# -- Fetches sources from local disk ------------------------------------------ 328 329class SourceFetcherForLocalFile(object): 330 def __init__(self, uri, fetchTarget): 331 self.fetchTarget = fetchTarget 332 self.uri = uri 333 self.sourceShouldBeValidated = False 334 335 def fetch(self): 336 # just symlink the local file to fetchTarget (if it exists) 337 portBaseDir = os.path.dirname(os.path.dirname(self.fetchTarget)) 338 localFile = portBaseDir + '/' + self.uri 339 if not os.path.isfile(localFile): 340 raise NameError("source %s doesn't exist" % localFile) 341 os.symlink(localFile, self.fetchTarget) 342 343 def updateToRev(self, rev): 344 pass 345 346 def unpack(self, sourceBaseDir, sourceSubDir, foldSubDir): 347 unpackFile(self.uri, self.fetchTarget, sourceBaseDir, sourceSubDir, 348 foldSubDir) 349 350# -- Fetches sources via hg --------------------------------------------------- 351 352class SourceFetcherForMercurial(object): 353 def __init__(self, uri, fetchTarget): 354 self.fetchTarget = fetchTarget 355 self.sourceShouldBeValidated = False 356 357 (unusedType, self.uri, self.rev) = parseCheckoutUri(uri) 358 359 def fetch(self): 360 if not Configuration.shallAllowUnsafeSources(): 361 sysExit('Downloading from unsafe sources is disabled in ' + 362 'haikuports.conf!') 363 364 warn("UNSAFE SOURCES ARE BAD AND SHOULD NOT BE USED IN PRODUCTION") 365 warn("PLEASE MOVE TO A STATIC ARCHIVE DOWNLOAD WITH CHECKSUM ASAP!") 366 ensureCommandIsAvailable('hg') 367 command = 'hg clone' 368 if self.rev: 369 command += ' -r ' + self.rev 370 command += ' ' + self.uri + ' ' + self.fetchTarget 371 output = check_output(command, shell=True, stderr=STDOUT).decode('utf-8') 372 info(output) 373 374 def updateToRev(self, rev): 375 ensureCommandIsAvailable('hg') 376 self.rev = rev 377 378 def unpack(self, sourceBaseDir, sourceSubDir, foldSubDir): 379 if not self.rev: 380 self.rev = 'tip' 381 382 sourceDir = sourceBaseDir + '/' + sourceSubDir \ 383 if sourceSubDir else sourceBaseDir 384 if foldSubDir: 385 command = 'hg archive -r %s -I "%s" -t files "%s"' \ 386 % (self.rev, foldSubDir, sourceDir) 387 else: 388 command = 'hg archive -r %s -t files "%s"' % (self.rev, sourceDir) 389 output = check_output(command, shell=True, cwd=self.fetchTarget).decode('utf-8') 390 info(output) 391 392 if foldSubDir: 393 foldSubdirIntoSourceDir(foldSubDir, sourceDir) 394 395# -- Fetches sources from source package -------------------------------------- 396 397class SourceFetcherForSourcePackage(object): 398 def __init__(self, uri, fetchTarget): 399 self.fetchTarget = fetchTarget 400 self.uri = uri 401 self.sourceShouldBeValidated = False 402 self.sourcePackagePath = self.uri[4:] 403 404 def fetch(self): 405 pass 406 407 def updateToRev(self, rev): 408 pass 409 410 def unpack(self, sourceBaseDir, sourceSubDir, foldSubDir): 411 sourceDir = sourceBaseDir + '/' + sourceSubDir \ 412 if sourceSubDir else sourceBaseDir 413 414 sourcePackageName = os.path.basename(self.sourcePackagePath) 415 (name, version, revision, unused) = sourcePackageName.split('-') 416 # determine port name by dropping '_source' or '_source_rigged' 417 if name.endswith('_source_rigged'): 418 name = name[:-14] 419 elif name.endswith('_source'): 420 name = name[:-7] 421 relativeSourcePath = ('develop/sources/%s-%s-%s/%s' 422 % (name, version, revision, 423 os.path.basename(sourceBaseDir))) 424 425 if not os.path.exists(sourceDir): 426 os.mkdir(sourceDir) 427 output = check_output([Configuration.getPackageCommand(), 'extract', 428 '-C', sourceDir, self.sourcePackagePath, 429 relativeSourcePath], stderr=STDOUT).decode('utf-8') 430 info(output) 431 foldSubdirIntoSourceDir(relativeSourcePath, sourceDir) 432 433# -- Fetches sources via svn -------------------------------------------------- 434 435class SourceFetcherForSubversion(object): 436 def __init__(self, uri, fetchTarget): 437 self.fetchTarget = fetchTarget 438 self.sourceShouldBeValidated = False 439 440 (unusedType, self.uri, self.rev) = parseCheckoutUri(uri) 441 442 def fetch(self): 443 if not Configuration.shallAllowUnsafeSources(): 444 sysExit('Downloading from unsafe sources is disabled in ' + 445 'haikuports.conf!') 446 447 ensureCommandIsAvailable('svn') 448 command = 'svn co --non-interactive --trust-server-cert' 449 if self.rev: 450 command += ' -r ' + self.rev 451 command += ' ' + self.uri + ' ' + self.fetchTarget 452 output = check_output(command, shell=True, stderr=STDOUT).decode('utf-8') 453 info(output) 454 455 def updateToRev(self, rev): 456 warn("Updating of a Subversion repository to a specific revision has " 457 u"not been implemented yet, sorry") 458 459 def unpack(self, sourceBaseDir, sourceSubDir, foldSubDir): 460 unpackCheckoutWithTar(self.fetchTarget, sourceBaseDir, sourceSubDir, 461 foldSubDir) 462 463# -- source fetcher factory function for given URI ---------------------------- 464 465def createSourceFetcher(uri, fetchTarget): 466 """Creates an appropriate source fetcher for the given URI""" 467 468 lowerUri = uri.lower() 469 if lowerUri.startswith('bzr'): 470 return SourceFetcherForBazaar(uri, fetchTarget) 471 elif lowerUri.startswith('cvs'): 472 return SourceFetcherForCvs(uri, fetchTarget) 473 elif lowerUri.startswith('fossil'): 474 return SourceFetcherForFossil(uri, fetchTarget) 475 elif lowerUri.startswith('git'): 476 return SourceFetcherForGit(uri, fetchTarget) 477 elif lowerUri.startswith('hg'): 478 return SourceFetcherForMercurial(uri, fetchTarget) 479 elif lowerUri.startswith('http') or lowerUri.startswith('ftp'): 480 return SourceFetcherForDownload(uri, fetchTarget) 481 elif lowerUri.startswith('pkg:'): 482 return SourceFetcherForSourcePackage(uri, fetchTarget) 483 elif lowerUri.startswith('svn'): 484 return SourceFetcherForSubversion(uri, fetchTarget) 485 elif lowerUri.startswith('file://'): 486 return SourceFetcherForLocalFile(uri[7:], fetchTarget) 487 elif ':' not in lowerUri: 488 return SourceFetcherForLocalFile(uri, fetchTarget) 489 else: 490 sysExit('The protocol of SOURCE_URI %s is unsupported, sorry.' % uri) 491