1254721Semaste/* Title: Pure/Tools/phabricator.scala 2254721Semaste Author: Makarius 3254721Semaste 4254721SemasteSupport for Phabricator server, notably for Ubuntu 18.04 LTS. 5254721Semaste 6254721SemasteSee also: 7254721Semaste - https://www.phacility.com/phabricator 8254721Semaste - https://secure.phabricator.com/book/phabricator 9254721Semaste*/ 10254721Semaste 11254721Semastepackage isabelle 12254721Semaste 13254721Semaste 14296417Sdimimport scala.collection.mutable 15296417Sdimimport scala.util.matching.Regex 16254721Semaste 17254721Semaste 18254721Semasteobject Phabricator 19254721Semaste{ 20254721Semaste /** defaults **/ 21254721Semaste 22254721Semaste /* required packages */ 23254721Semaste 24254721Semaste val packages: List[String] = 25254721Semaste Build_Docker.packages ::: Linux.packages ::: 26254721Semaste List( 27254721Semaste // https://secure.phabricator.com/source/phabricator/browse/master/scripts/install/install_ubuntu.sh 15e6e2adea61 28254721Semaste "git", "mysql-server", "apache2", "libapache2-mod-php", "php", "php-mysql", 29254721Semaste "php-gd", "php-curl", "php-apcu", "php-cli", "php-json", "php-mbstring", 30254721Semaste // more packages 31254721Semaste "php-xml", "php-zip", "python-pygments", "ssh", "subversion", 32254721Semaste // mercurial build packages 33254721Semaste "make", "gcc", "python", "python-dev", "python-docutils", "python-pygments", "python-openssl") 34254721Semaste 35254721Semaste 36254721Semaste /* global system resources */ 37254721Semaste 38254721Semaste val www_user = "www-data" 39254721Semaste 40254721Semaste val daemon_user = "phabricator" 41254721Semaste 42254721Semaste val sshd_config = Path.explode("/etc/ssh/sshd_config") 43254721Semaste 44254721Semaste 45254721Semaste /* installation parameters */ 46254721Semaste 47254721Semaste val default_name = "vcs" 48254721Semaste 49254721Semaste def phabricator_name(name: String = "", ext: String = ""): String = 50254721Semaste "phabricator" + (if (name.isEmpty) "" else "-" + name) + (if (ext.isEmpty) "" else "." + ext) 51254721Semaste 52254721Semaste def isabelle_phabricator_name(name: String = "", ext: String = ""): String = 53254721Semaste "isabelle-" + phabricator_name(name = name, ext = ext) 54254721Semaste 55254721Semaste def default_root(name: String): Path = 56254721Semaste Path.explode("/var/www") + Path.basic(phabricator_name(name = name)) 57254721Semaste 58254721Semaste def default_repo(name: String): Path = default_root(name) + Path.basic("repo") 59254721Semaste 60254721Semaste val default_mailers: Path = Path.explode("mailers.json") 61296417Sdim 62254721Semaste val default_system_port = SSH.default_port 63254721Semaste val alternative_system_port = 222 64254721Semaste val default_server_port = 2222 65254721Semaste 66254721Semaste val standard_mercurial_source = "https://www.mercurial-scm.org/release/mercurial-3.9.2.tar.gz" 67254721Semaste 68254721Semaste 69254721Semaste 70254721Semaste /** global configuration **/ 71254721Semaste 72254721Semaste val global_config = Path.explode("/etc/" + isabelle_phabricator_name(ext = "conf")) 73254721Semaste 74254721Semaste def global_config_script( 75254721Semaste init: String = "", 76254721Semaste body: String = "", 77254721Semaste exit: String = ""): String = 78254721Semaste { 79296417Sdim"""#!/bin/bash 80254721Semaste""" + (if (init.nonEmpty) "\n" + init else "") + """ 81254721Semaste{ 82254721Semaste while { unset REPLY; read -r; test "$?" = 0 -o -n "$REPLY"; } 83254721Semaste do 84254721Semaste NAME="$(echo "$REPLY" | cut -d: -f1)" 85254721Semaste ROOT="$(echo "$REPLY" | cut -d: -f2)" 86254721Semaste { 87254721Semaste""" + Library.prefix_lines(" ", body) + """ 88254721Semaste } < /dev/null 89254721Semaste done 90254721Semaste} < """ + File.bash_path(global_config) + "\n" + 91254721Semaste (if (exit.nonEmpty) "\n" + exit + "\n" else "") 92254721Semaste } 93254721Semaste 94254721Semaste sealed case class Config(name: String, root: Path) 95254721Semaste { 96254721Semaste def home: Path = root + Path.explode(phabricator_name()) 97254721Semaste 98254721Semaste def execute(command: String): Process_Result = 99254721Semaste Isabelle_System.bash("bin/" + command, cwd = home.file, redirect = true).check 100254721Semaste } 101254721Semaste 102254721Semaste def read_config(): List[Config] = 103254721Semaste { 104296417Sdim if (global_config.is_file) { 105254721Semaste for (entry <- Library.trim_split_lines(File.read(global_config)) if entry.nonEmpty) 106254721Semaste yield { 107254721Semaste space_explode(':', entry) match { 108254721Semaste case List(name, root) => Config(name, Path.explode(root)) 109254721Semaste case _ => error("Malformed config file " + global_config + "\nentry " + quote(entry)) 110254721Semaste } 111254721Semaste } 112254721Semaste } 113254721Semaste else Nil 114254721Semaste } 115254721Semaste 116254721Semaste def write_config(configs: List[Config]) 117254721Semaste { 118254721Semaste File.write(global_config, 119254721Semaste configs.map(config => config.name + ":" + config.root.implode).mkString("", "\n", "\n")) 120254721Semaste } 121254721Semaste 122254721Semaste def get_config(name: String): Config = 123254721Semaste read_config().find(config => config.name == name) getOrElse 124254721Semaste error("Bad Isabelle/Phabricator installation " + quote(name)) 125254721Semaste 126254721Semaste 127254721Semaste 128254721Semaste /** administrative tools **/ 129254721Semaste 130254721Semaste /* Isabelle tool wrapper */ 131254721Semaste 132254721Semaste val isabelle_tool1 = 133254721Semaste Isabelle_Tool("phabricator", "invoke command-line tool within Phabricator home directory", args => 134254721Semaste { 135254721Semaste var list = false 136254721Semaste var name = default_name 137254721Semaste 138254721Semaste val getopts = 139254721Semaste Getopts(""" 140254721SemasteUsage: isabelle phabricator [OPTIONS] COMMAND [ARGS...] 141254721Semaste 142254721Semaste Options are: 143254721Semaste -l list available Phabricator installations 144296417Sdim -n NAME Phabricator installation name (default: """ + quote(default_name) + """) 145296417Sdim 146296417Sdim Invoke a command-line tool within the home directory of the named 147296417Sdim Phabricator installation. 148254721Semaste""", 149254721Semaste "l" -> (_ => list = true), 150254721Semaste "n:" -> (arg => name = arg)) 151254721Semaste 152254721Semaste val more_args = getopts(args) 153254721Semaste if (more_args.isEmpty && !list) getopts.usage() 154276479Sdim 155254721Semaste val progress = new Console_Progress 156254721Semaste 157254721Semaste if (list) { 158254721Semaste for (config <- read_config()) { 159254721Semaste progress.echo("phabricator " + quote(config.name) + " root " + config.root) 160254721Semaste } 161254721Semaste } 162254721Semaste else { 163254721Semaste val config = get_config(name) 164254721Semaste val result = progress.bash(Bash.strings(more_args), cwd = config.home.file, echo = true) 165254721Semaste if (!result.ok) error("Return code: " + result.rc.toString) 166254721Semaste } 167254721Semaste }) 168254721Semaste 169254721Semaste 170254721Semaste 171254721Semaste /** setup **/ 172254721Semaste 173254721Semaste def user_setup(name: String, description: String, ssh_setup: Boolean = false) 174254721Semaste { 175254721Semaste if (!Linux.user_exists(name)) { 176254721Semaste Linux.user_add(name, description = description, system = true, ssh_setup = ssh_setup) 177254721Semaste } 178254721Semaste else if (Linux.user_description(name) != description) { 179254721Semaste error("User " + quote(name) + " already exists --" + 180254721Semaste " for Phabricator it should have the description:\n " + quote(description)) 181254721Semaste } 182296417Sdim } 183254721Semaste 184254721Semaste def command_setup(name: String, 185254721Semaste init: String = "", 186254721Semaste body: String = "", 187254721Semaste exit: String = ""): Path = 188254721Semaste { 189254721Semaste val command = Path.explode("/usr/local/bin") + Path.basic(name) 190254721Semaste File.write(command, global_config_script(init = init, body = body, exit = exit)) 191254721Semaste Isabelle_System.chmod("755", command) 192254721Semaste Isabelle_System.chown("root:root", command) 193254721Semaste command 194254721Semaste } 195296417Sdim 196254721Semaste def mercurial_setup(mercurial_source: String, progress: Progress = No_Progress) 197254721Semaste { 198254721Semaste progress.echo("\nMercurial installation from source " + quote(mercurial_source) + " ...") 199254721Semaste Isabelle_System.with_tmp_dir("mercurial")(tmp_dir => 200254721Semaste { 201254721Semaste val archive = 202254721Semaste if (Url.is_wellformed(mercurial_source)) { 203254721Semaste val archive = tmp_dir + Path.basic("mercurial.tar.gz") 204254721Semaste Bytes.write(archive, Url.read_bytes(Url(mercurial_source))) 205254721Semaste archive 206254721Semaste } 207254721Semaste else Path.explode(mercurial_source) 208254721Semaste 209254721Semaste Isabelle_System.gnutar("-xzf " + File.bash_path(archive), dir = tmp_dir).check 210254721Semaste 211254721Semaste File.read_dir(tmp_dir).filter(name => (tmp_dir + Path.basic(name)).is_dir) match { 212254721Semaste case List(dir) => 213296417Sdim val build_dir = tmp_dir + Path.basic(dir) 214254721Semaste progress.bash("make all && make install", cwd = build_dir.file, echo = true).check 215254721Semaste case dirs => 216254721Semaste error("Bad archive " + archive + 217254721Semaste (if (dirs.isEmpty) "" else "\nmultiple directory entries " + commas_quote(dirs))) 218254721Semaste } 219254721Semaste }) 220254721Semaste } 221254721Semaste 222254721Semaste def phabricator_setup( 223254721Semaste options: Options, 224254721Semaste name: String = default_name, 225254721Semaste root: String = "", 226254721Semaste repo: String = "", 227254721Semaste package_update: Boolean = false, 228254721Semaste mercurial_source: String = "", 229254721Semaste progress: Progress = No_Progress) 230254721Semaste { 231254721Semaste /* system environment */ 232254721Semaste 233254721Semaste Linux.check_system_root() 234254721Semaste 235254721Semaste progress.echo("System packages ...") 236280031Sdim 237254721Semaste if (package_update) { 238254721Semaste Linux.package_update(progress = progress) 239254721Semaste Linux.check_reboot_required() 240254721Semaste } 241254721Semaste 242254721Semaste Linux.package_install(packages, progress = progress) 243254721Semaste Linux.check_reboot_required() 244254721Semaste 245254721Semaste 246254721Semaste if (mercurial_source.nonEmpty) { 247254721Semaste for { name <- List("mercurial", "mercurial-common") if Linux.package_installed(name) } { 248254721Semaste error("Cannot install Mercurial from source:\n" + 249254721Semaste "package package " + quote(name) + " already installed") 250254721Semaste } 251254721Semaste mercurial_setup(mercurial_source, progress = progress) 252254721Semaste } 253254721Semaste 254254721Semaste 255254721Semaste /* users */ 256254721Semaste 257254721Semaste if (name.contains((c: Char) => !(Symbol.is_ascii_letter(c) || Symbol.is_ascii_digit(c))) || 258254721Semaste Set("", "ssh", "phd", "dump", daemon_user).contains(name)) { 259254721Semaste error("Bad installation name: " + quote(name)) 260254721Semaste } 261254721Semaste 262254721Semaste user_setup(daemon_user, "Phabricator Daemon User", ssh_setup = true) 263254721Semaste user_setup(name, "Phabricator SSH User") 264254721Semaste 265254721Semaste 266254721Semaste /* basic installation */ 267254721Semaste 268254721Semaste progress.echo("\nPhabricator installation ...") 269254721Semaste 270254721Semaste val root_path = if (root.nonEmpty) Path.explode(root) else default_root(name) 271254721Semaste val repo_path = if (repo.nonEmpty) Path.explode(repo) else default_repo(name) 272254721Semaste 273254721Semaste val configs = read_config() 274254721Semaste 275254721Semaste for (config <- configs if config.name == name) { 276254721Semaste error("Duplicate Phabricator installation " + quote(name) + " in " + config.root) 277254721Semaste } 278254721Semaste 279254721Semaste if (!Isabelle_System.bash("mkdir -p " + File.bash_path(root_path)).ok) { 280254721Semaste error("Failed to create root directory " + root_path) 281254721Semaste } 282254721Semaste 283254721Semaste Isabelle_System.chown(Bash.string(www_user) + ":" + Bash.string(www_user), root_path) 284254721Semaste Isabelle_System.chmod("755", root_path) 285254721Semaste 286254721Semaste progress.bash(cwd = root_path.file, echo = true, 287254721Semaste script = """ 288254721Semaste set -e 289254721Semaste echo "Cloning distribution repositories:" 290254721Semaste 291254721Semaste git clone --branch stable https://github.com/phacility/arcanist.git 292254721Semaste git -C arcanist reset --hard """ + 293 Bash.string(options.string("phabricator_version_arcanist")) + """ 294 295 git clone --branch stable https://github.com/phacility/libphutil.git 296 git -C libphutil reset --hard """ + 297 Bash.string(options.string("phabricator_version_libphutil")) + """ 298 299 git clone --branch stable https://github.com/phacility/phabricator.git 300 git -C phabricator reset --hard """ + 301 Bash.string(options.string("phabricator_version_phabricator")) + """ 302 """).check 303 304 val config = Config(name, root_path) 305 write_config(configs ::: List(config)) 306 307 config.execute("config set pygments.enabled true") 308 309 310 /* local repository directory */ 311 312 progress.echo("\nRepository hosting setup ...") 313 314 if (!Isabelle_System.bash("mkdir -p " + File.bash_path(repo_path)).ok) { 315 error("Failed to create local repository directory " + repo_path) 316 } 317 318 Isabelle_System.chown( 319 "-R " + Bash.string(daemon_user) + ":" + Bash.string(daemon_user), repo_path) 320 Isabelle_System.chmod("755", repo_path) 321 322 config.execute("config set repository.default-local-path " + File.bash_path(repo_path)) 323 324 325 val sudoers_file = 326 Path.explode("/etc/sudoers.d") + Path.basic(isabelle_phabricator_name(name = name)) 327 File.write(sudoers_file, 328 www_user + " ALL=(" + daemon_user + ") SETENV: NOPASSWD: /usr/bin/git, /usr/local/bin/hg, /usr/bin/hg, /usr/bin/ssh, /usr/bin/id\n" + 329 name + " ALL=(" + daemon_user + ") SETENV: NOPASSWD: /usr/bin/git, /usr/bin/git-upload-pack, /usr/bin/git-receive-pack, /usr/local/bin/hg, /usr/bin/hg, /usr/bin/svnserve, /usr/bin/ssh, /usr/bin/id\n") 330 331 Isabelle_System.chmod("440", sudoers_file) 332 333 config.execute("config set diffusion.ssh-user " + Bash.string(config.name)) 334 335 336 /* MySQL setup */ 337 338 progress.echo("\nMySQL setup ...") 339 340 File.write(Path.explode("/etc/mysql/mysql.conf.d/" + phabricator_name(ext = "cnf")), 341"""[mysqld] 342max_allowed_packet = 32M 343innodb_buffer_pool_size = 1600M 344local_infile = 0 345""") 346 347 Linux.service_restart("mysql") 348 349 350 def mysql_conf(R: Regex, which: String): String = 351 { 352 val conf = Path.explode("/etc/mysql/debian.cnf") 353 split_lines(File.read(conf)).collectFirst({ case R(a) => a }) match { 354 case Some(res) => res 355 case None => error("Cannot determine " + which + " from " + conf) 356 } 357 } 358 359 val mysql_root_user = mysql_conf("""^user\s*=\s*(\S*)\s*$""".r, "superuser name") 360 val mysql_root_password = mysql_conf("""^password\s*=\s*(\S*)\s*$""".r, "superuser password") 361 362 val mysql_name = phabricator_name(name = name).replace("-", "_") 363 val mysql_user_string = SQL.string(mysql_name) + "@'localhost'" 364 val mysql_password = Linux.generate_password() 365 366 Isabelle_System.bash("mysql --user=" + Bash.string(mysql_root_user) + 367 " --password=" + Bash.string(mysql_root_password) + " --execute=" + 368 Bash.string( 369 """DROP USER IF EXISTS """ + mysql_user_string + "; " + 370 """CREATE USER """ + mysql_user_string + 371 """ IDENTIFIED BY """ + SQL.string(mysql_password) + """ PASSWORD EXPIRE NEVER; """ + 372 """GRANT ALL ON `""" + (mysql_name + "_%").replace("_", "\\_") + 373 """`.* TO """ + mysql_user_string + ";")).check 374 375 config.execute("config set mysql.user " + Bash.string(mysql_name)) 376 config.execute("config set mysql.pass " + Bash.string(mysql_password)) 377 378 config.execute("config set phabricator.cache-namespace " + Bash.string(mysql_name)) 379 config.execute("config set storage.default-namespace " + Bash.string(mysql_name)) 380 config.execute("config set storage.mysql-engine.max-size 8388608") 381 382 progress.bash("bin/storage upgrade --force", cwd = config.home.file, echo = true).check 383 384 385 /* database dump */ 386 387 val dump_name = isabelle_phabricator_name(name = "dump") 388 command_setup(dump_name, body = 389"""mkdir -p "$ROOT/database" && chown root:root "$ROOT/database" && chmod 700 "$ROOT/database" 390[ -e "$ROOT/database/dump.sql.gz" ] && mv -f "$ROOT/database/dump.sql.gz" "$ROOT/database/dump-old.sql.gz" 391echo -n "Creating $ROOT/database/dump.sql.gz ..." 392"$ROOT/phabricator/bin/storage" dump --compress --output "$ROOT/database/dump.sql.gz" 2>&1 | fgrep -v '[Warning] Using a password on the command line interface can be insecure' 393echo " $(ls -hs "$ROOT/database/dump.sql.gz" | cut -d" " -f1)" """) 394 395 396 /* Phabricator upgrade */ 397 398 command_setup(isabelle_phabricator_name(name = "upgrade"), 399 init = 400"""BRANCH="${1:-stable}" 401if [ "$BRANCH" != "master" -a "$BRANCH" != "stable" ] 402then 403 echo "Bad branch: \"$BRANCH\"" 404 exit 1 405fi 406 407systemctl stop isabelle-phabricator-phd 408systemctl stop apache2 409""", 410 body = 411"""echo -e "\nUpgrading phabricator \"$NAME\" root \"$ROOT\" ..." 412for REPO in libphutil arcanist phabricator 413do 414 cd "$ROOT/$REPO" 415 echo -e "\nUpdating \"$REPO\" ..." 416 git checkout "$BRANCH" 417 git pull 418done 419echo -e "\nUpgrading storage ..." 420"$ROOT/phabricator/bin/storage" upgrade --force 421""", 422 exit = 423"""systemctl start apache2 424systemctl start isabelle-phabricator-phd""") 425 426 427 /* PHP setup */ 428 429 val php_version = 430 Isabelle_System.bash("""php --run 'echo PHP_MAJOR_VERSION . "." . PHP_MINOR_VERSION;'""") 431 .check.out 432 433 val php_conf = 434 Path.explode("/etc/php") + Path.basic(php_version) + // educated guess 435 Path.explode("apache2/conf.d") + 436 Path.basic(isabelle_phabricator_name(ext = "ini")) 437 438 File.write(php_conf, 439 "post_max_size = 32M\n" + 440 "opcache.validate_timestamps = 0\n" + 441 "memory_limit = 512M\n" + 442 "max_execution_time = 120\n") 443 444 445 /* Apache setup */ 446 447 progress.echo("Apache setup ...") 448 449 val apache_root = Path.explode("/etc/apache2") 450 val apache_sites = apache_root + Path.explode("sites-available") 451 452 if (!apache_sites.is_dir) error("Bad Apache sites directory " + apache_sites) 453 454 val server_name = phabricator_name(name = name, ext = "lvh.me") // alias for "localhost" for testing 455 val server_url = "http://" + server_name 456 457 File.write(apache_sites + Path.basic(isabelle_phabricator_name(name = name, ext = "conf")), 458"""<VirtualHost *:80> 459 ServerName """ + server_name + """ 460 ServerAdmin webmaster@localhost 461 DocumentRoot """ + config.home.implode + """/webroot 462 463 ErrorLog ${APACHE_LOG_DIR}/error.log 464 CustomLog ${APACHE_LOG_DIR}/access.log combined 465 466 RewriteEngine on 467 RewriteRule ^(.*)$ /index.php?__path__=$1 [B,L,QSA] 468</VirtualHost> 469 470# vim: syntax=apache ts=4 sw=4 sts=4 sr noet 471""") 472 473 Isabelle_System.bash( """ 474 set -e 475 a2enmod rewrite 476 a2ensite """ + Bash.string(isabelle_phabricator_name(name = name))).check 477 478 config.execute("config set phabricator.base-uri " + Bash.string(server_url)) 479 480 Linux.service_restart("apache2") 481 482 progress.echo("\nFurther manual configuration via " + server_url) 483 484 485 /* PHP daemon */ 486 487 progress.echo("\nPHP daemon setup ...") 488 489 val phd_log_path = Path.explode("/var/tmp/phd") 490 Isabelle_System.mkdirs(phd_log_path) 491 Isabelle_System.chown( 492 "-R " + Bash.string(daemon_user) + ":" + Bash.string(daemon_user), phd_log_path) 493 Isabelle_System.chmod("755", phd_log_path) 494 495 config.execute("config set phd.user " + Bash.string(daemon_user)) 496 config.execute("config set phd.log-directory /var/tmp/phd/" + 497 isabelle_phabricator_name(name = name) + "/log") 498 499 val phd_name = isabelle_phabricator_name(name = "phd") 500 Linux.service_shutdown(phd_name) 501 val phd_command = command_setup(phd_name, body = """"$ROOT/phabricator/bin/phd" "$@" """) 502 try { 503 Linux.service_install(phd_name, 504"""[Unit] 505Description=PHP daemon manager for Isabelle/Phabricator 506After=syslog.target network.target apache2.service mysql.service 507 508[Service] 509Type=oneshot 510User=""" + daemon_user + """ 511Group=""" + daemon_user + """ 512Environment=PATH=/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin:/usr/bin:/bin 513ExecStart=""" + phd_command.implode + """ start --force 514ExecStop=""" + phd_command.implode + """ stop 515RemainAfterExit=yes 516 517[Install] 518WantedBy=multi-user.target 519""") 520 } 521 catch { 522 case ERROR(msg) => 523 progress.bash("bin/phd status", cwd = config.home.file, echo = true).check 524 error(msg) 525 } 526 } 527 528 529 /* Isabelle tool wrapper */ 530 531 val isabelle_tool2 = 532 Isabelle_Tool("phabricator_setup", "setup Phabricator server on Ubuntu Linux", args => 533 { 534 var mercurial_source = "" 535 var repo = "" 536 var package_update = false 537 var name = default_name 538 var options = Options.init() 539 var root = "" 540 541 val getopts = 542 Getopts(""" 543Usage: isabelle phabricator_setup [OPTIONS] 544 545 Options are: 546 -M SOURCE install Mercurial from source: local PATH, or URL, or ":" for 547 """ + standard_mercurial_source + """ 548 -R DIR repository directory (default: """ + default_repo("NAME") + """) 549 -U full update of system packages before installation 550 -n NAME Phabricator installation name (default: """ + quote(default_name) + """) 551 -o OPTION override Isabelle system OPTION (via NAME=VAL or NAME) 552 -r DIR installation root directory (default: """ + default_root("NAME") + """) 553 554 Install Phabricator as LAMP application (Linux, Apache, MySQL, PHP). 555 556 The installation name (default: """ + quote(default_name) + """) is mapped to a regular 557 Unix user; this is relevant for public SSH access. 558""", 559 "M:" -> (arg => mercurial_source = (if (arg == ":") standard_mercurial_source else arg)), 560 "R:" -> (arg => repo = arg), 561 "U" -> (_ => package_update = true), 562 "n:" -> (arg => name = arg), 563 "o:" -> (arg => options = options + arg), 564 "r:" -> (arg => root = arg)) 565 566 val more_args = getopts(args) 567 if (more_args.nonEmpty) getopts.usage() 568 569 val progress = new Console_Progress 570 571 val release = Linux.Release() 572 if (!release.is_ubuntu_18_04) error("Bad Linux version: Ubuntu 18.04 LTS required") 573 574 phabricator_setup(options, name = name, root = root, repo = repo, 575 package_update = package_update, mercurial_source = mercurial_source, progress = progress) 576 }) 577 578 579 580 /** setup mail **/ 581 582 val mailers_template: String = 583"""[ 584 { 585 "key": "example.org", 586 "type": "smtp", 587 "options": { 588 "host": "mail.example.org", 589 "port": 465, 590 "user": "phabricator@example.org", 591 "password": "********", 592 "protocol": "ssl", 593 "message-id": true 594 } 595 } 596]""" 597 598 def phabricator_setup_mail( 599 name: String = default_name, 600 config_file: Option[Path] = None, 601 test_user: String = "", 602 progress: Progress = No_Progress) 603 { 604 Linux.check_system_root() 605 606 val config = get_config(name) 607 val default_config_file = config.root + default_mailers 608 609 val mail_config = config_file getOrElse default_config_file 610 611 def setup_mail 612 { 613 progress.echo("Using mail configuration from " + mail_config) 614 config.execute("config set cluster.mailers --stdin < " + File.bash_path(mail_config)) 615 616 if (test_user.nonEmpty) { 617 progress.echo("Sending test mail to " + quote(test_user)) 618 progress.bash(cwd = config.home.file, echo = true, 619 script = """echo "Test from Phabricator ($(date))" | bin/mail send-test --subject "Test" --to """ + 620 Bash.string(test_user)).check 621 } 622 } 623 624 if (config_file.isEmpty) { 625 if (!default_config_file.is_file) { 626 File.write(default_config_file, mailers_template) 627 Isabelle_System.chmod("600", default_config_file) 628 } 629 if (File.read(default_config_file) == mailers_template) { 630 progress.echo("Please invoke the tool again, after providing details in\n " + 631 default_config_file.implode + "\n") 632 } 633 else setup_mail 634 } 635 else setup_mail 636 } 637 638 639 /* Isabelle tool wrapper */ 640 641 val isabelle_tool3 = 642 Isabelle_Tool("phabricator_setup_mail", 643 "setup mail for one Phabricator installation", args => 644 { 645 var test_user = "" 646 var name = default_name 647 var config_file: Option[Path] = None 648 649 val getopts = 650 Getopts(""" 651Usage: isabelle phabricator_setup_mail [OPTIONS] 652 653 Options are: 654 -T USER send test mail to Phabricator user 655 -f FILE config file (default: """ + default_mailers + """ within Phabricator root) 656 -n NAME Phabricator installation name (default: """ + quote(default_name) + """) 657 658 Provide mail configuration for existing Phabricator installation. 659""", 660 "T:" -> (arg => test_user = arg), 661 "f:" -> (arg => config_file = Some(Path.explode(arg))), 662 "n:" -> (arg => name = arg)) 663 664 val more_args = getopts(args) 665 if (more_args.nonEmpty) getopts.usage() 666 667 val progress = new Console_Progress 668 669 phabricator_setup_mail(name = name, config_file = config_file, 670 test_user = test_user, progress = progress) 671 }) 672 673 674 675 /** setup ssh **/ 676 677 /* sshd config */ 678 679 private val Port = """^\s*Port\s+(\d+)\s*$""".r 680 private val No_Port = """^#\s*Port\b.*$""".r 681 private val Any_Port = """^#?\s*Port\b.*$""".r 682 683 def conf_ssh_port(port: Int): String = 684 if (port == SSH.default_port) "#Port " + SSH.default_port else "Port " + port 685 686 def read_ssh_port(conf: Path): Int = 687 { 688 val lines = split_lines(File.read(conf)) 689 val ports = 690 lines.flatMap({ 691 case Port(Value.Int(p)) => Some(p) 692 case No_Port() => Some(SSH.default_port) 693 case _ => None 694 }) 695 ports match { 696 case List(port) => port 697 case Nil => error("Missing Port specification in " + conf) 698 case _ => error("Multiple Port specifications in " + conf) 699 } 700 } 701 702 def write_ssh_port(conf: Path, port: Int): Boolean = 703 { 704 val old_port = read_ssh_port(conf) 705 if (old_port == port) false 706 else { 707 val lines = split_lines(File.read(conf)) 708 val lines1 = lines.map({ case Any_Port() => conf_ssh_port(port) case line => line }) 709 File.write(conf, cat_lines(lines1)) 710 true 711 } 712 } 713 714 715 /* phabricator_setup_ssh */ 716 717 def phabricator_setup_ssh( 718 server_port: Int = default_server_port, 719 system_port: Int = default_system_port, 720 progress: Progress = No_Progress) 721 { 722 Linux.check_system_root() 723 724 val configs = read_config() 725 726 if (server_port == system_port) { 727 error("Port for Phabricator sshd coincides with system port: " + system_port) 728 } 729 730 val sshd_conf_system = Path.explode("/etc/ssh/sshd_config") 731 val sshd_conf_server = sshd_conf_system.ext(isabelle_phabricator_name()) 732 733 val ssh_name = isabelle_phabricator_name(name = "ssh") 734 Linux.service_shutdown(ssh_name) 735 736 val old_system_port = read_ssh_port(sshd_conf_system) 737 if (old_system_port != system_port) { 738 progress.echo("Reconfigurig system ssh service") 739 Linux.service_shutdown("ssh") 740 write_ssh_port(sshd_conf_system, system_port) 741 Linux.service_start("ssh") 742 } 743 744 progress.echo("Configuring " + ssh_name + " service") 745 746 val ssh_command = command_setup(ssh_name, body = 747"""if [ "$1" = "$NAME" ] 748then 749 exec "$ROOT/phabricator/bin/ssh-auth" "$@" 750fi""", exit = "exit 1") 751 752 File.write(sshd_conf_server, 753"""# OpenBSD Secure Shell server for Isabelle/Phabricator 754AuthorizedKeysCommand """ + ssh_command.implode + """ 755AuthorizedKeysCommandUser """ + daemon_user + """ 756AuthorizedKeysFile none 757AllowUsers """ + configs.map(_.name).mkString(" ") + """ 758Port """ + server_port + """ 759Protocol 2 760PermitRootLogin no 761AllowAgentForwarding no 762AllowTcpForwarding no 763PrintMotd no 764PrintLastLog no 765PasswordAuthentication no 766ChallengeResponseAuthentication no 767PidFile /var/run/""" + ssh_name + """.pid 768""") 769 770 Linux.service_install(ssh_name, 771"""[Unit] 772Description=OpenBSD Secure Shell server for Isabelle/Phabricator 773After=network.target auditd.service 774ConditionPathExists=!/etc/ssh/sshd_not_to_be_run 775 776[Service] 777EnvironmentFile=-/etc/default/ssh 778ExecStartPre=/usr/sbin/sshd -f """ + sshd_conf_server.implode + """ -t 779ExecStart=/usr/sbin/sshd -f """ + sshd_conf_server.implode + """ -D $SSHD_OPTS 780ExecReload=/usr/sbin/sshd -f """ + sshd_conf_server.implode + """ -t 781ExecReload=/bin/kill -HUP $MAINPID 782KillMode=process 783Restart=on-failure 784RestartPreventExitStatus=255 785Type=notify 786RuntimeDirectory=sshd-phabricator 787RuntimeDirectoryMode=0755 788 789[Install] 790WantedBy=multi-user.target 791Alias=""" + ssh_name + """.service 792""") 793 794 for (config <- configs) { 795 progress.echo("phabricator " + quote(config.name) + " port " + server_port) 796 config.execute("config set diffusion.ssh-port " + Bash.string(server_port.toString)) 797 if (server_port == SSH.default_port) config.execute("config delete diffusion.ssh-port") 798 } 799 } 800 801 802 /* Isabelle tool wrapper */ 803 804 val isabelle_tool4 = 805 Isabelle_Tool("phabricator_setup_ssh", 806 "setup ssh service for all Phabricator installations", args => 807 { 808 var server_port = default_server_port 809 var system_port = default_system_port 810 811 val getopts = 812 Getopts(""" 813Usage: isabelle phabricator_setup_ssh [OPTIONS] 814 815 Options are: 816 -p PORT sshd port for Phabricator servers (default: """ + default_server_port + """) 817 -q PORT sshd port for the operating system (default: """ + default_system_port + """) 818 819 Configure ssh service for all Phabricator installations: a separate sshd 820 is run in addition to the one of the operating system, and ports need to 821 be distinct. 822 823 A particular Phabricator installation is addressed by using its 824 name as the ssh user; the actual Phabricator user is determined via 825 stored ssh keys. 826""", 827 "p:" -> (arg => server_port = Value.Int.parse(arg)), 828 "q:" -> (arg => system_port = Value.Int.parse(arg))) 829 830 val more_args = getopts(args) 831 if (more_args.nonEmpty) getopts.usage() 832 833 val progress = new Console_Progress 834 835 phabricator_setup_ssh( 836 server_port = server_port, system_port = system_port, progress = progress) 837 }) 838 839 840 841 /** conduit API **/ 842 843 object API 844 { 845 /* user information */ 846 847 sealed case class User( 848 id: Long, 849 phid: String, 850 name: String, 851 real_name: String, 852 roles: List[String]) 853 { 854 def is_valid: Boolean = 855 roles.contains("verified") && 856 roles.contains("approved") && 857 roles.contains("activated") 858 def is_admin: Boolean = roles.contains("admin") 859 def is_regular: Boolean = !(roles.contains("bot") || roles.contains("list")) 860 } 861 862 863 /* repository information */ 864 865 sealed case class Repository( 866 vcs: VCS.Value, 867 id: Long, 868 phid: String, 869 name: String, 870 callsign: String, 871 short_name: String, 872 importing: Boolean, 873 ssh_url: String) 874 { 875 def is_hg: Boolean = vcs == VCS.hg 876 } 877 878 object VCS extends Enumeration 879 { 880 val hg, git, svn = Value 881 def read(s: String): Value = 882 try { withName(s) } 883 catch { case _: java.util.NoSuchElementException => error("Unknown vcs type " + quote(s)) } 884 } 885 886 def edits(typ: String, value: JSON.T): List[JSON.Object.T] = 887 List(JSON.Object("type" -> typ, "value" -> value)) 888 889 def opt_edits(typ: String, value: Option[JSON.T]): List[JSON.Object.T] = 890 value.toList.flatMap(edits(typ, _)) 891 892 893 /* result with optional error */ 894 895 sealed case class Result(result: JSON.T, error: Option[String]) 896 { 897 def ok: Boolean = error.isEmpty 898 def get: JSON.T = if (ok) result else Exn.error(error.get) 899 900 def get_value[A](unapply: JSON.T => Option[A]): A = 901 unapply(get) getOrElse Exn.error("Bad JSON result: " + JSON.Format(result)) 902 903 def get_string: String = get_value(JSON.Value.String.unapply) 904 } 905 906 def make_result(json: JSON.T): Result = 907 { 908 val result = JSON.value(json, "result").getOrElse(JSON.Object.empty) 909 val error_info = JSON.string(json, "error_info") 910 val error_code = JSON.string(json, "error_code") 911 Result(result, error_info orElse error_code) 912 } 913 914 915 /* context for operations */ 916 917 def apply(user: String, host: String, port: Int = SSH.default_port): API = 918 new API(user, host, port) 919 } 920 921 final class API private(ssh_user: String, ssh_host: String, ssh_port: Int) 922 { 923 /* connection */ 924 925 require(ssh_host.nonEmpty && ssh_port >= 0) 926 927 private def ssh_user_prefix: String = SSH.user_prefix(ssh_user) 928 private def ssh_port_suffix: String = SSH.port_suffix(ssh_port) 929 930 override def toString: String = ssh_user_prefix + ssh_host + ssh_port_suffix 931 def hg_url: String = "ssh://" + ssh_user_prefix + ssh_host + ssh_port_suffix 932 933 934 /* execute methods */ 935 936 def execute_raw(method: String, params: JSON.T = JSON.Object.empty): JSON.T = 937 { 938 Isabelle_System.with_tmp_file("params", "json")(params_file => 939 { 940 File.write(params_file, JSON.Format(JSON.Object("params" -> JSON.Format(params)))) 941 val result = 942 Isabelle_System.bash( 943 "ssh -p " + ssh_port + " " + Bash.string(ssh_user_prefix + ssh_host) + 944 " conduit " + Bash.string(method) + " < " + File.bash_path(params_file)).check 945 JSON.parse(result.out, strict = false) 946 }) 947 } 948 949 def execute(method: String, params: JSON.T = JSON.Object.empty): API.Result = 950 API.make_result(execute_raw(method, params = params)) 951 952 def execute_search[A]( 953 method: String, params: JSON.Object.T, unapply: JSON.T => Option[A]): List[A] = 954 { 955 val results = new mutable.ListBuffer[A] 956 var after = "" 957 958 do { 959 val result = 960 execute(method, params = params ++ JSON.optional("after" -> proper_string(after))) 961 results ++= result.get_value(JSON.list(_, "data", unapply)) 962 after = result.get_value(JSON.value(_, "cursor", JSON.string0(_, "after"))) 963 } while (after.nonEmpty) 964 965 results.toList 966 } 967 968 def ping(): String = execute("conduit.ping").get_string 969 970 971 /* users */ 972 973 lazy val user_phid: String = execute("user.whoami").get_value(JSON.string(_, "phid")) 974 lazy val user_name: String = execute("user.whoami").get_value(JSON.string(_, "userName")) 975 976 def get_users( 977 all: Boolean = false, 978 phid: String = "", 979 name: String = ""): List[API.User] = 980 { 981 val constraints: JSON.Object.T = 982 (for { (key, value) <- List("phids" -> phid, "usernames" -> name) if value.nonEmpty } 983 yield (key, List(value))).toMap 984 985 execute_search("user.search", 986 JSON.Object("queryKey" -> (if (all) "all" else "active"), "constraints" -> constraints), 987 data => JSON.value(data, "fields", fields => 988 for { 989 id <- JSON.long(data, "id") 990 phid <- JSON.string(data, "phid") 991 name <- JSON.string(fields, "username") 992 real_name <- JSON.string0(fields, "realName") 993 roles <- JSON.strings(fields, "roles") 994 } yield API.User(id, phid, name, real_name, roles))) 995 } 996 997 def the_user(phid: String): API.User = 998 get_users(phid = phid) match { 999 case List(user) => user 1000 case _ => error("Bad user PHID " + quote(phid)) 1001 } 1002 1003 1004 /* repositories */ 1005 1006 def get_repositories( 1007 all: Boolean = false, 1008 phid: String = "", 1009 callsign: String = "", 1010 short_name: String = ""): List[API.Repository] = 1011 { 1012 val constraints: JSON.Object.T = 1013 (for { 1014 (key, value) <- List("phids" -> phid, "callsigns" -> callsign, "shortNames" -> short_name) 1015 if value.nonEmpty 1016 } yield (key, List(value))).toMap 1017 1018 execute_search("diffusion.repository.search", 1019 JSON.Object("queryKey" -> (if (all) "all" else "active"), "constraints" -> constraints), 1020 data => JSON.value(data, "fields", fields => 1021 for { 1022 vcs_name <- JSON.string(fields, "vcs") 1023 id <- JSON.long(data, "id") 1024 phid <- JSON.string(data, "phid") 1025 name <- JSON.string(fields, "name") 1026 callsign <- JSON.string0(fields, "callsign") 1027 short_name <- JSON.string0(fields, "shortName") 1028 importing <- JSON.bool(fields, "isImporting") 1029 } 1030 yield { 1031 val vcs = API.VCS.read(vcs_name) 1032 val url_path = 1033 if (short_name.isEmpty) "/diffusion/" + id else "/source/" + short_name 1034 val ssh_url = 1035 vcs match { 1036 case API.VCS.hg => hg_url + url_path 1037 case API.VCS.git => hg_url + url_path + ".git" 1038 case API.VCS.svn => "" 1039 } 1040 API.Repository(vcs, id, phid, name, callsign, short_name, importing, ssh_url) 1041 })) 1042 } 1043 1044 def the_repository(phid: String): API.Repository = 1045 get_repositories(phid = phid) match { 1046 case List(repo) => repo 1047 case _ => error("Bad repository PHID " + quote(phid)) 1048 } 1049 1050 def create_repository( 1051 name: String, 1052 callsign: String = "", // unique name, UPPERCASE 1053 short_name: String = "", // unique name 1054 description: String = "", 1055 public: Boolean = false, 1056 vcs: API.VCS.Value = API.VCS.hg): API.Repository = 1057 { 1058 require(name.nonEmpty) 1059 1060 val transactions = 1061 API.edits("vcs", vcs.toString) ::: 1062 API.edits("name", name) ::: 1063 API.opt_edits("callsign", proper_string(callsign)) ::: 1064 API.opt_edits("shortName", proper_string(short_name)) ::: 1065 API.opt_edits("description", proper_string(description)) ::: 1066 (if (public) Nil 1067 else API.edits("view", user_phid) ::: API.edits("policy.push", user_phid)) ::: 1068 API.edits("status", "active") 1069 1070 val phid = 1071 execute("diffusion.repository.edit", params = JSON.Object("transactions" -> transactions)) 1072 .get_value(JSON.value(_, "object", JSON.string(_, "phid"))) 1073 1074 execute("diffusion.looksoon", params = JSON.Object("repositories" -> List(phid))).get 1075 1076 the_repository(phid) 1077 } 1078 } 1079} 1080