1/* Title: Pure/Tools/phabricator.scala 2 Author: Makarius 3 4Support for Phabricator server, notably for Ubuntu 18.04 LTS. 5 6See also: 7 - https://www.phacility.com/phabricator 8 - https://secure.phabricator.com/book/phabricator 9*/ 10 11package isabelle 12 13 14import scala.collection.mutable 15import scala.util.matching.Regex 16 17 18object Phabricator 19{ 20 /** defaults **/ 21 22 /* required packages */ 23 24 val packages: List[String] = 25 Build_Docker.packages ::: Linux.packages ::: 26 List( 27 // https://secure.phabricator.com/source/phabricator/browse/master/scripts/install/install_ubuntu.sh 15e6e2adea61 28 "git", "mysql-server", "apache2", "libapache2-mod-php", "php", "php-mysql", 29 "php-gd", "php-curl", "php-apcu", "php-cli", "php-json", "php-mbstring", 30 // more packages 31 "php-xml", "php-zip", "python-pygments", "ssh", "subversion", 32 // mercurial build packages 33 "make", "gcc", "python", "python-dev", "python-docutils", "python-pygments", "python-openssl") 34 35 36 /* global system resources */ 37 38 val www_user = "www-data" 39 40 val daemon_user = "phabricator" 41 42 val sshd_config = Path.explode("/etc/ssh/sshd_config") 43 44 45 /* installation parameters */ 46 47 val default_name = "vcs" 48 49 def phabricator_name(name: String = "", ext: String = ""): String = 50 "phabricator" + (if (name.isEmpty) "" else "-" + name) + (if (ext.isEmpty) "" else "." + ext) 51 52 def isabelle_phabricator_name(name: String = "", ext: String = ""): String = 53 "isabelle-" + phabricator_name(name = name, ext = ext) 54 55 def default_root(name: String): Path = 56 Path.explode("/var/www") + Path.basic(phabricator_name(name = name)) 57 58 def default_repo(name: String): Path = default_root(name) + Path.basic("repo") 59 60 val default_mailers: Path = Path.explode("mailers.json") 61 62 val default_system_port = SSH.default_port 63 val alternative_system_port = 222 64 val default_server_port = 2222 65 66 val standard_mercurial_source = "https://www.mercurial-scm.org/release/mercurial-3.9.2.tar.gz" 67 68 69 70 /** global configuration **/ 71 72 val global_config = Path.explode("/etc/" + isabelle_phabricator_name(ext = "conf")) 73 74 def global_config_script( 75 init: String = "", 76 body: String = "", 77 exit: String = ""): String = 78 { 79"""#!/bin/bash 80""" + (if (init.nonEmpty) "\n" + init else "") + """ 81{ 82 while { unset REPLY; read -r; test "$?" = 0 -o -n "$REPLY"; } 83 do 84 NAME="$(echo "$REPLY" | cut -d: -f1)" 85 ROOT="$(echo "$REPLY" | cut -d: -f2)" 86 { 87""" + Library.prefix_lines(" ", body) + """ 88 } < /dev/null 89 done 90} < """ + File.bash_path(global_config) + "\n" + 91 (if (exit.nonEmpty) "\n" + exit + "\n" else "") 92 } 93 94 sealed case class Config(name: String, root: Path) 95 { 96 def home: Path = root + Path.explode(phabricator_name()) 97 98 def execute(command: String): Process_Result = 99 Isabelle_System.bash("bin/" + command, cwd = home.file, redirect = true).check 100 } 101 102 def read_config(): List[Config] = 103 { 104 if (global_config.is_file) { 105 for (entry <- Library.trim_split_lines(File.read(global_config)) if entry.nonEmpty) 106 yield { 107 space_explode(':', entry) match { 108 case List(name, root) => Config(name, Path.explode(root)) 109 case _ => error("Malformed config file " + global_config + "\nentry " + quote(entry)) 110 } 111 } 112 } 113 else Nil 114 } 115 116 def write_config(configs: List[Config]) 117 { 118 File.write(global_config, 119 configs.map(config => config.name + ":" + config.root.implode).mkString("", "\n", "\n")) 120 } 121 122 def get_config(name: String): Config = 123 read_config().find(config => config.name == name) getOrElse 124 error("Bad Isabelle/Phabricator installation " + quote(name)) 125 126 127 128 /** administrative tools **/ 129 130 /* Isabelle tool wrapper */ 131 132 val isabelle_tool1 = 133 Isabelle_Tool("phabricator", "invoke command-line tool within Phabricator home directory", args => 134 { 135 var list = false 136 var name = default_name 137 138 val getopts = 139 Getopts(""" 140Usage: isabelle phabricator [OPTIONS] COMMAND [ARGS...] 141 142 Options are: 143 -l list available Phabricator installations 144 -n NAME Phabricator installation name (default: """ + quote(default_name) + """) 145 146 Invoke a command-line tool within the home directory of the named 147 Phabricator installation. 148""", 149 "l" -> (_ => list = true), 150 "n:" -> (arg => name = arg)) 151 152 val more_args = getopts(args) 153 if (more_args.isEmpty && !list) getopts.usage() 154 155 val progress = new Console_Progress 156 157 if (list) { 158 for (config <- read_config()) { 159 progress.echo("phabricator " + quote(config.name) + " root " + config.root) 160 } 161 } 162 else { 163 val config = get_config(name) 164 val result = progress.bash(Bash.strings(more_args), cwd = config.home.file, echo = true) 165 if (!result.ok) error("Return code: " + result.rc.toString) 166 } 167 }) 168 169 170 171 /** setup **/ 172 173 def user_setup(name: String, description: String, ssh_setup: Boolean = false) 174 { 175 if (!Linux.user_exists(name)) { 176 Linux.user_add(name, description = description, system = true, ssh_setup = ssh_setup) 177 } 178 else if (Linux.user_description(name) != description) { 179 error("User " + quote(name) + " already exists --" + 180 " for Phabricator it should have the description:\n " + quote(description)) 181 } 182 } 183 184 def command_setup(name: String, 185 init: String = "", 186 body: String = "", 187 exit: String = ""): Path = 188 { 189 val command = Path.explode("/usr/local/bin") + Path.basic(name) 190 File.write(command, global_config_script(init = init, body = body, exit = exit)) 191 Isabelle_System.chmod("755", command) 192 Isabelle_System.chown("root:root", command) 193 command 194 } 195 196 def mercurial_setup(mercurial_source: String, progress: Progress = No_Progress) 197 { 198 progress.echo("\nMercurial installation from source " + quote(mercurial_source) + " ...") 199 Isabelle_System.with_tmp_dir("mercurial")(tmp_dir => 200 { 201 val archive = 202 if (Url.is_wellformed(mercurial_source)) { 203 val archive = tmp_dir + Path.basic("mercurial.tar.gz") 204 Bytes.write(archive, Url.read_bytes(Url(mercurial_source))) 205 archive 206 } 207 else Path.explode(mercurial_source) 208 209 Isabelle_System.gnutar("-xzf " + File.bash_path(archive), dir = tmp_dir).check 210 211 File.read_dir(tmp_dir).filter(name => (tmp_dir + Path.basic(name)).is_dir) match { 212 case List(dir) => 213 val build_dir = tmp_dir + Path.basic(dir) 214 progress.bash("make all && make install", cwd = build_dir.file, echo = true).check 215 case dirs => 216 error("Bad archive " + archive + 217 (if (dirs.isEmpty) "" else "\nmultiple directory entries " + commas_quote(dirs))) 218 } 219 }) 220 } 221 222 def phabricator_setup( 223 options: Options, 224 name: String = default_name, 225 root: String = "", 226 repo: String = "", 227 package_update: Boolean = false, 228 mercurial_source: String = "", 229 progress: Progress = No_Progress) 230 { 231 /* system environment */ 232 233 Linux.check_system_root() 234 235 progress.echo("System packages ...") 236 237 if (package_update) { 238 Linux.package_update(progress = progress) 239 Linux.check_reboot_required() 240 } 241 242 Linux.package_install(packages, progress = progress) 243 Linux.check_reboot_required() 244 245 246 if (mercurial_source.nonEmpty) { 247 for { name <- List("mercurial", "mercurial-common") if Linux.package_installed(name) } { 248 error("Cannot install Mercurial from source:\n" + 249 "package package " + quote(name) + " already installed") 250 } 251 mercurial_setup(mercurial_source, progress = progress) 252 } 253 254 255 /* users */ 256 257 if (name.contains((c: Char) => !(Symbol.is_ascii_letter(c) || Symbol.is_ascii_digit(c))) || 258 Set("", "ssh", "phd", "dump", daemon_user).contains(name)) { 259 error("Bad installation name: " + quote(name)) 260 } 261 262 user_setup(daemon_user, "Phabricator Daemon User", ssh_setup = true) 263 user_setup(name, "Phabricator SSH User") 264 265 266 /* basic installation */ 267 268 progress.echo("\nPhabricator installation ...") 269 270 val root_path = if (root.nonEmpty) Path.explode(root) else default_root(name) 271 val repo_path = if (repo.nonEmpty) Path.explode(repo) else default_repo(name) 272 273 val configs = read_config() 274 275 for (config <- configs if config.name == name) { 276 error("Duplicate Phabricator installation " + quote(name) + " in " + config.root) 277 } 278 279 if (!Isabelle_System.bash("mkdir -p " + File.bash_path(root_path)).ok) { 280 error("Failed to create root directory " + root_path) 281 } 282 283 Isabelle_System.chown(Bash.string(www_user) + ":" + Bash.string(www_user), root_path) 284 Isabelle_System.chmod("755", root_path) 285 286 progress.bash(cwd = root_path.file, echo = true, 287 script = """ 288 set -e 289 echo "Cloning distribution repositories:" 290 291 git clone --branch stable https://github.com/phacility/arcanist.git 292 git -C arcanist reset --hard """ + 293 Bash.string(options.string("phabricator_version_arcanist")) + """ 294 295 git clone --branch stable https://github.com/phacility/libphutil.git 296 git -C libphutil reset --hard """ + 297 Bash.string(options.string("phabricator_version_libphutil")) + """ 298 299 git clone --branch stable https://github.com/phacility/phabricator.git 300 git -C phabricator reset --hard """ + 301 Bash.string(options.string("phabricator_version_phabricator")) + """ 302 """).check 303 304 val config = Config(name, root_path) 305 write_config(configs ::: List(config)) 306 307 config.execute("config set pygments.enabled true") 308 309 310 /* local repository directory */ 311 312 progress.echo("\nRepository hosting setup ...") 313 314 if (!Isabelle_System.bash("mkdir -p " + File.bash_path(repo_path)).ok) { 315 error("Failed to create local repository directory " + repo_path) 316 } 317 318 Isabelle_System.chown( 319 "-R " + Bash.string(daemon_user) + ":" + Bash.string(daemon_user), repo_path) 320 Isabelle_System.chmod("755", repo_path) 321 322 config.execute("config set repository.default-local-path " + File.bash_path(repo_path)) 323 324 325 val sudoers_file = 326 Path.explode("/etc/sudoers.d") + Path.basic(isabelle_phabricator_name(name = name)) 327 File.write(sudoers_file, 328 www_user + " ALL=(" + daemon_user + ") SETENV: NOPASSWD: /usr/bin/git, /usr/local/bin/hg, /usr/bin/hg, /usr/bin/ssh, /usr/bin/id\n" + 329 name + " ALL=(" + daemon_user + ") SETENV: NOPASSWD: /usr/bin/git, /usr/bin/git-upload-pack, /usr/bin/git-receive-pack, /usr/local/bin/hg, /usr/bin/hg, /usr/bin/svnserve, /usr/bin/ssh, /usr/bin/id\n") 330 331 Isabelle_System.chmod("440", sudoers_file) 332 333 config.execute("config set diffusion.ssh-user " + Bash.string(config.name)) 334 335 336 /* MySQL setup */ 337 338 progress.echo("\nMySQL setup ...") 339 340 File.write(Path.explode("/etc/mysql/mysql.conf.d/" + phabricator_name(ext = "cnf")), 341"""[mysqld] 342max_allowed_packet = 32M 343innodb_buffer_pool_size = 1600M 344local_infile = 0 345""") 346 347 Linux.service_restart("mysql") 348 349 350 def mysql_conf(R: Regex, which: String): String = 351 { 352 val conf = Path.explode("/etc/mysql/debian.cnf") 353 split_lines(File.read(conf)).collectFirst({ case R(a) => a }) match { 354 case Some(res) => res 355 case None => error("Cannot determine " + which + " from " + conf) 356 } 357 } 358 359 val mysql_root_user = mysql_conf("""^user\s*=\s*(\S*)\s*$""".r, "superuser name") 360 val mysql_root_password = mysql_conf("""^password\s*=\s*(\S*)\s*$""".r, "superuser password") 361 362 val mysql_name = phabricator_name(name = name).replace("-", "_") 363 val mysql_user_string = SQL.string(mysql_name) + "@'localhost'" 364 val mysql_password = Linux.generate_password() 365 366 Isabelle_System.bash("mysql --user=" + Bash.string(mysql_root_user) + 367 " --password=" + Bash.string(mysql_root_password) + " --execute=" + 368 Bash.string( 369 """DROP USER IF EXISTS """ + mysql_user_string + "; " + 370 """CREATE USER """ + mysql_user_string + 371 """ IDENTIFIED BY """ + SQL.string(mysql_password) + """ PASSWORD EXPIRE NEVER; """ + 372 """GRANT ALL ON `""" + (mysql_name + "_%").replace("_", "\\_") + 373 """`.* TO """ + mysql_user_string + ";")).check 374 375 config.execute("config set mysql.user " + Bash.string(mysql_name)) 376 config.execute("config set mysql.pass " + Bash.string(mysql_password)) 377 378 config.execute("config set phabricator.cache-namespace " + Bash.string(mysql_name)) 379 config.execute("config set storage.default-namespace " + Bash.string(mysql_name)) 380 config.execute("config set storage.mysql-engine.max-size 8388608") 381 382 progress.bash("bin/storage upgrade --force", cwd = config.home.file, echo = true).check 383 384 385 /* database dump */ 386 387 val dump_name = isabelle_phabricator_name(name = "dump") 388 command_setup(dump_name, body = 389"""mkdir -p "$ROOT/database" && chown root:root "$ROOT/database" && chmod 700 "$ROOT/database" 390[ -e "$ROOT/database/dump.sql.gz" ] && mv -f "$ROOT/database/dump.sql.gz" "$ROOT/database/dump-old.sql.gz" 391echo -n "Creating $ROOT/database/dump.sql.gz ..." 392"$ROOT/phabricator/bin/storage" dump --compress --output "$ROOT/database/dump.sql.gz" 2>&1 | fgrep -v '[Warning] Using a password on the command line interface can be insecure' 393echo " $(ls -hs "$ROOT/database/dump.sql.gz" | cut -d" " -f1)" """) 394 395 396 /* Phabricator upgrade */ 397 398 command_setup(isabelle_phabricator_name(name = "upgrade"), 399 init = 400"""BRANCH="${1:-stable}" 401if [ "$BRANCH" != "master" -a "$BRANCH" != "stable" ] 402then 403 echo "Bad branch: \"$BRANCH\"" 404 exit 1 405fi 406 407systemctl stop isabelle-phabricator-phd 408systemctl stop apache2 409""", 410 body = 411"""echo -e "\nUpgrading phabricator \"$NAME\" root \"$ROOT\" ..." 412for REPO in libphutil arcanist phabricator 413do 414 cd "$ROOT/$REPO" 415 echo -e "\nUpdating \"$REPO\" ..." 416 git checkout "$BRANCH" 417 git pull 418done 419echo -e "\nUpgrading storage ..." 420"$ROOT/phabricator/bin/storage" upgrade --force 421""", 422 exit = 423"""systemctl start apache2 424systemctl start isabelle-phabricator-phd""") 425 426 427 /* PHP setup */ 428 429 val php_version = 430 Isabelle_System.bash("""php --run 'echo PHP_MAJOR_VERSION . "." . PHP_MINOR_VERSION;'""") 431 .check.out 432 433 val php_conf = 434 Path.explode("/etc/php") + Path.basic(php_version) + // educated guess 435 Path.explode("apache2/conf.d") + 436 Path.basic(isabelle_phabricator_name(ext = "ini")) 437 438 File.write(php_conf, 439 "post_max_size = 32M\n" + 440 "opcache.validate_timestamps = 0\n" + 441 "memory_limit = 512M\n" + 442 "max_execution_time = 120\n") 443 444 445 /* Apache setup */ 446 447 progress.echo("Apache setup ...") 448 449 val apache_root = Path.explode("/etc/apache2") 450 val apache_sites = apache_root + Path.explode("sites-available") 451 452 if (!apache_sites.is_dir) error("Bad Apache sites directory " + apache_sites) 453 454 val server_name = phabricator_name(name = name, ext = "lvh.me") // alias for "localhost" for testing 455 val server_url = "http://" + server_name 456 457 File.write(apache_sites + Path.basic(isabelle_phabricator_name(name = name, ext = "conf")), 458"""<VirtualHost *:80> 459 ServerName """ + server_name + """ 460 ServerAdmin webmaster@localhost 461 DocumentRoot """ + config.home.implode + """/webroot 462 463 ErrorLog ${APACHE_LOG_DIR}/error.log 464 CustomLog ${APACHE_LOG_DIR}/access.log combined 465 466 RewriteEngine on 467 RewriteRule ^(.*)$ /index.php?__path__=$1 [B,L,QSA] 468</VirtualHost> 469 470# vim: syntax=apache ts=4 sw=4 sts=4 sr noet 471""") 472 473 Isabelle_System.bash( """ 474 set -e 475 a2enmod rewrite 476 a2ensite """ + Bash.string(isabelle_phabricator_name(name = name))).check 477 478 config.execute("config set phabricator.base-uri " + Bash.string(server_url)) 479 480 Linux.service_restart("apache2") 481 482 progress.echo("\nFurther manual configuration via " + server_url) 483 484 485 /* PHP daemon */ 486 487 progress.echo("\nPHP daemon setup ...") 488 489 val phd_log_path = Path.explode("/var/tmp/phd") 490 Isabelle_System.mkdirs(phd_log_path) 491 Isabelle_System.chown( 492 "-R " + Bash.string(daemon_user) + ":" + Bash.string(daemon_user), phd_log_path) 493 Isabelle_System.chmod("755", phd_log_path) 494 495 config.execute("config set phd.user " + Bash.string(daemon_user)) 496 config.execute("config set phd.log-directory /var/tmp/phd/" + 497 isabelle_phabricator_name(name = name) + "/log") 498 499 val phd_name = isabelle_phabricator_name(name = "phd") 500 Linux.service_shutdown(phd_name) 501 val phd_command = command_setup(phd_name, body = """"$ROOT/phabricator/bin/phd" "$@" """) 502 try { 503 Linux.service_install(phd_name, 504"""[Unit] 505Description=PHP daemon manager for Isabelle/Phabricator 506After=syslog.target network.target apache2.service mysql.service 507 508[Service] 509Type=oneshot 510User=""" + daemon_user + """ 511Group=""" + daemon_user + """ 512Environment=PATH=/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin:/usr/bin:/bin 513ExecStart=""" + phd_command.implode + """ start --force 514ExecStop=""" + phd_command.implode + """ stop 515RemainAfterExit=yes 516 517[Install] 518WantedBy=multi-user.target 519""") 520 } 521 catch { 522 case ERROR(msg) => 523 progress.bash("bin/phd status", cwd = config.home.file, echo = true).check 524 error(msg) 525 } 526 } 527 528 529 /* Isabelle tool wrapper */ 530 531 val isabelle_tool2 = 532 Isabelle_Tool("phabricator_setup", "setup Phabricator server on Ubuntu Linux", args => 533 { 534 var mercurial_source = "" 535 var repo = "" 536 var package_update = false 537 var name = default_name 538 var options = Options.init() 539 var root = "" 540 541 val getopts = 542 Getopts(""" 543Usage: isabelle phabricator_setup [OPTIONS] 544 545 Options are: 546 -M SOURCE install Mercurial from source: local PATH, or URL, or ":" for 547 """ + standard_mercurial_source + """ 548 -R DIR repository directory (default: """ + default_repo("NAME") + """) 549 -U full update of system packages before installation 550 -n NAME Phabricator installation name (default: """ + quote(default_name) + """) 551 -o OPTION override Isabelle system OPTION (via NAME=VAL or NAME) 552 -r DIR installation root directory (default: """ + default_root("NAME") + """) 553 554 Install Phabricator as LAMP application (Linux, Apache, MySQL, PHP). 555 556 The installation name (default: """ + quote(default_name) + """) is mapped to a regular 557 Unix user; this is relevant for public SSH access. 558""", 559 "M:" -> (arg => mercurial_source = (if (arg == ":") standard_mercurial_source else arg)), 560 "R:" -> (arg => repo = arg), 561 "U" -> (_ => package_update = true), 562 "n:" -> (arg => name = arg), 563 "o:" -> (arg => options = options + arg), 564 "r:" -> (arg => root = arg)) 565 566 val more_args = getopts(args) 567 if (more_args.nonEmpty) getopts.usage() 568 569 val progress = new Console_Progress 570 571 val release = Linux.Release() 572 if (!release.is_ubuntu_18_04) error("Bad Linux version: Ubuntu 18.04 LTS required") 573 574 phabricator_setup(options, name = name, root = root, repo = repo, 575 package_update = package_update, mercurial_source = mercurial_source, progress = progress) 576 }) 577 578 579 580 /** setup mail **/ 581 582 val mailers_template: String = 583"""[ 584 { 585 "key": "example.org", 586 "type": "smtp", 587 "options": { 588 "host": "mail.example.org", 589 "port": 465, 590 "user": "phabricator@example.org", 591 "password": "********", 592 "protocol": "ssl", 593 "message-id": true 594 } 595 } 596]""" 597 598 def phabricator_setup_mail( 599 name: String = default_name, 600 config_file: Option[Path] = None, 601 test_user: String = "", 602 progress: Progress = No_Progress) 603 { 604 Linux.check_system_root() 605 606 val config = get_config(name) 607 val default_config_file = config.root + default_mailers 608 609 val mail_config = config_file getOrElse default_config_file 610 611 def setup_mail 612 { 613 progress.echo("Using mail configuration from " + mail_config) 614 config.execute("config set cluster.mailers --stdin < " + File.bash_path(mail_config)) 615 616 if (test_user.nonEmpty) { 617 progress.echo("Sending test mail to " + quote(test_user)) 618 progress.bash(cwd = config.home.file, echo = true, 619 script = """echo "Test from Phabricator ($(date))" | bin/mail send-test --subject "Test" --to """ + 620 Bash.string(test_user)).check 621 } 622 } 623 624 if (config_file.isEmpty) { 625 if (!default_config_file.is_file) { 626 File.write(default_config_file, mailers_template) 627 Isabelle_System.chmod("600", default_config_file) 628 } 629 if (File.read(default_config_file) == mailers_template) { 630 progress.echo("Please invoke the tool again, after providing details in\n " + 631 default_config_file.implode + "\n") 632 } 633 else setup_mail 634 } 635 else setup_mail 636 } 637 638 639 /* Isabelle tool wrapper */ 640 641 val isabelle_tool3 = 642 Isabelle_Tool("phabricator_setup_mail", 643 "setup mail for one Phabricator installation", args => 644 { 645 var test_user = "" 646 var name = default_name 647 var config_file: Option[Path] = None 648 649 val getopts = 650 Getopts(""" 651Usage: isabelle phabricator_setup_mail [OPTIONS] 652 653 Options are: 654 -T USER send test mail to Phabricator user 655 -f FILE config file (default: """ + default_mailers + """ within Phabricator root) 656 -n NAME Phabricator installation name (default: """ + quote(default_name) + """) 657 658 Provide mail configuration for existing Phabricator installation. 659""", 660 "T:" -> (arg => test_user = arg), 661 "f:" -> (arg => config_file = Some(Path.explode(arg))), 662 "n:" -> (arg => name = arg)) 663 664 val more_args = getopts(args) 665 if (more_args.nonEmpty) getopts.usage() 666 667 val progress = new Console_Progress 668 669 phabricator_setup_mail(name = name, config_file = config_file, 670 test_user = test_user, progress = progress) 671 }) 672 673 674 675 /** setup ssh **/ 676 677 /* sshd config */ 678 679 private val Port = """^\s*Port\s+(\d+)\s*$""".r 680 private val No_Port = """^#\s*Port\b.*$""".r 681 private val Any_Port = """^#?\s*Port\b.*$""".r 682 683 def conf_ssh_port(port: Int): String = 684 if (port == SSH.default_port) "#Port " + SSH.default_port else "Port " + port 685 686 def read_ssh_port(conf: Path): Int = 687 { 688 val lines = split_lines(File.read(conf)) 689 val ports = 690 lines.flatMap({ 691 case Port(Value.Int(p)) => Some(p) 692 case No_Port() => Some(SSH.default_port) 693 case _ => None 694 }) 695 ports match { 696 case List(port) => port 697 case Nil => error("Missing Port specification in " + conf) 698 case _ => error("Multiple Port specifications in " + conf) 699 } 700 } 701 702 def write_ssh_port(conf: Path, port: Int): Boolean = 703 { 704 val old_port = read_ssh_port(conf) 705 if (old_port == port) false 706 else { 707 val lines = split_lines(File.read(conf)) 708 val lines1 = lines.map({ case Any_Port() => conf_ssh_port(port) case line => line }) 709 File.write(conf, cat_lines(lines1)) 710 true 711 } 712 } 713 714 715 /* phabricator_setup_ssh */ 716 717 def phabricator_setup_ssh( 718 server_port: Int = default_server_port, 719 system_port: Int = default_system_port, 720 progress: Progress = No_Progress) 721 { 722 Linux.check_system_root() 723 724 val configs = read_config() 725 726 if (server_port == system_port) { 727 error("Port for Phabricator sshd coincides with system port: " + system_port) 728 } 729 730 val sshd_conf_system = Path.explode("/etc/ssh/sshd_config") 731 val sshd_conf_server = sshd_conf_system.ext(isabelle_phabricator_name()) 732 733 val ssh_name = isabelle_phabricator_name(name = "ssh") 734 Linux.service_shutdown(ssh_name) 735 736 val old_system_port = read_ssh_port(sshd_conf_system) 737 if (old_system_port != system_port) { 738 progress.echo("Reconfigurig system ssh service") 739 Linux.service_shutdown("ssh") 740 write_ssh_port(sshd_conf_system, system_port) 741 Linux.service_start("ssh") 742 } 743 744 progress.echo("Configuring " + ssh_name + " service") 745 746 val ssh_command = command_setup(ssh_name, body = 747"""if [ "$1" = "$NAME" ] 748then 749 exec "$ROOT/phabricator/bin/ssh-auth" "$@" 750fi""", exit = "exit 1") 751 752 File.write(sshd_conf_server, 753"""# OpenBSD Secure Shell server for Isabelle/Phabricator 754AuthorizedKeysCommand """ + ssh_command.implode + """ 755AuthorizedKeysCommandUser """ + daemon_user + """ 756AuthorizedKeysFile none 757AllowUsers """ + configs.map(_.name).mkString(" ") + """ 758Port """ + server_port + """ 759Protocol 2 760PermitRootLogin no 761AllowAgentForwarding no 762AllowTcpForwarding no 763PrintMotd no 764PrintLastLog no 765PasswordAuthentication no 766ChallengeResponseAuthentication no 767PidFile /var/run/""" + ssh_name + """.pid 768""") 769 770 Linux.service_install(ssh_name, 771"""[Unit] 772Description=OpenBSD Secure Shell server for Isabelle/Phabricator 773After=network.target auditd.service 774ConditionPathExists=!/etc/ssh/sshd_not_to_be_run 775 776[Service] 777EnvironmentFile=-/etc/default/ssh 778ExecStartPre=/usr/sbin/sshd -f """ + sshd_conf_server.implode + """ -t 779ExecStart=/usr/sbin/sshd -f """ + sshd_conf_server.implode + """ -D $SSHD_OPTS 780ExecReload=/usr/sbin/sshd -f """ + sshd_conf_server.implode + """ -t 781ExecReload=/bin/kill -HUP $MAINPID 782KillMode=process 783Restart=on-failure 784RestartPreventExitStatus=255 785Type=notify 786RuntimeDirectory=sshd-phabricator 787RuntimeDirectoryMode=0755 788 789[Install] 790WantedBy=multi-user.target 791Alias=""" + ssh_name + """.service 792""") 793 794 for (config <- configs) { 795 progress.echo("phabricator " + quote(config.name) + " port " + server_port) 796 config.execute("config set diffusion.ssh-port " + Bash.string(server_port.toString)) 797 if (server_port == SSH.default_port) config.execute("config delete diffusion.ssh-port") 798 } 799 } 800 801 802 /* Isabelle tool wrapper */ 803 804 val isabelle_tool4 = 805 Isabelle_Tool("phabricator_setup_ssh", 806 "setup ssh service for all Phabricator installations", args => 807 { 808 var server_port = default_server_port 809 var system_port = default_system_port 810 811 val getopts = 812 Getopts(""" 813Usage: isabelle phabricator_setup_ssh [OPTIONS] 814 815 Options are: 816 -p PORT sshd port for Phabricator servers (default: """ + default_server_port + """) 817 -q PORT sshd port for the operating system (default: """ + default_system_port + """) 818 819 Configure ssh service for all Phabricator installations: a separate sshd 820 is run in addition to the one of the operating system, and ports need to 821 be distinct. 822 823 A particular Phabricator installation is addressed by using its 824 name as the ssh user; the actual Phabricator user is determined via 825 stored ssh keys. 826""", 827 "p:" -> (arg => server_port = Value.Int.parse(arg)), 828 "q:" -> (arg => system_port = Value.Int.parse(arg))) 829 830 val more_args = getopts(args) 831 if (more_args.nonEmpty) getopts.usage() 832 833 val progress = new Console_Progress 834 835 phabricator_setup_ssh( 836 server_port = server_port, system_port = system_port, progress = progress) 837 }) 838 839 840 841 /** conduit API **/ 842 843 object API 844 { 845 /* user information */ 846 847 sealed case class User( 848 id: Long, 849 phid: String, 850 name: String, 851 real_name: String, 852 roles: List[String]) 853 { 854 def is_valid: Boolean = 855 roles.contains("verified") && 856 roles.contains("approved") && 857 roles.contains("activated") 858 def is_admin: Boolean = roles.contains("admin") 859 def is_regular: Boolean = !(roles.contains("bot") || roles.contains("list")) 860 } 861 862 863 /* repository information */ 864 865 sealed case class Repository( 866 vcs: VCS.Value, 867 id: Long, 868 phid: String, 869 name: String, 870 callsign: String, 871 short_name: String, 872 importing: Boolean, 873 ssh_url: String) 874 { 875 def is_hg: Boolean = vcs == VCS.hg 876 } 877 878 object VCS extends Enumeration 879 { 880 val hg, git, svn = Value 881 def read(s: String): Value = 882 try { withName(s) } 883 catch { case _: java.util.NoSuchElementException => error("Unknown vcs type " + quote(s)) } 884 } 885 886 def edits(typ: String, value: JSON.T): List[JSON.Object.T] = 887 List(JSON.Object("type" -> typ, "value" -> value)) 888 889 def opt_edits(typ: String, value: Option[JSON.T]): List[JSON.Object.T] = 890 value.toList.flatMap(edits(typ, _)) 891 892 893 /* result with optional error */ 894 895 sealed case class Result(result: JSON.T, error: Option[String]) 896 { 897 def ok: Boolean = error.isEmpty 898 def get: JSON.T = if (ok) result else Exn.error(error.get) 899 900 def get_value[A](unapply: JSON.T => Option[A]): A = 901 unapply(get) getOrElse Exn.error("Bad JSON result: " + JSON.Format(result)) 902 903 def get_string: String = get_value(JSON.Value.String.unapply) 904 } 905 906 def make_result(json: JSON.T): Result = 907 { 908 val result = JSON.value(json, "result").getOrElse(JSON.Object.empty) 909 val error_info = JSON.string(json, "error_info") 910 val error_code = JSON.string(json, "error_code") 911 Result(result, error_info orElse error_code) 912 } 913 914 915 /* context for operations */ 916 917 def apply(user: String, host: String, port: Int = SSH.default_port): API = 918 new API(user, host, port) 919 } 920 921 final class API private(ssh_user: String, ssh_host: String, ssh_port: Int) 922 { 923 /* connection */ 924 925 require(ssh_host.nonEmpty && ssh_port >= 0) 926 927 private def ssh_user_prefix: String = SSH.user_prefix(ssh_user) 928 private def ssh_port_suffix: String = SSH.port_suffix(ssh_port) 929 930 override def toString: String = ssh_user_prefix + ssh_host + ssh_port_suffix 931 def hg_url: String = "ssh://" + ssh_user_prefix + ssh_host + ssh_port_suffix 932 933 934 /* execute methods */ 935 936 def execute_raw(method: String, params: JSON.T = JSON.Object.empty): JSON.T = 937 { 938 Isabelle_System.with_tmp_file("params", "json")(params_file => 939 { 940 File.write(params_file, JSON.Format(JSON.Object("params" -> JSON.Format(params)))) 941 val result = 942 Isabelle_System.bash( 943 "ssh -p " + ssh_port + " " + Bash.string(ssh_user_prefix + ssh_host) + 944 " conduit " + Bash.string(method) + " < " + File.bash_path(params_file)).check 945 JSON.parse(result.out, strict = false) 946 }) 947 } 948 949 def execute(method: String, params: JSON.T = JSON.Object.empty): API.Result = 950 API.make_result(execute_raw(method, params = params)) 951 952 def execute_search[A]( 953 method: String, params: JSON.Object.T, unapply: JSON.T => Option[A]): List[A] = 954 { 955 val results = new mutable.ListBuffer[A] 956 var after = "" 957 958 do { 959 val result = 960 execute(method, params = params ++ JSON.optional("after" -> proper_string(after))) 961 results ++= result.get_value(JSON.list(_, "data", unapply)) 962 after = result.get_value(JSON.value(_, "cursor", JSON.string0(_, "after"))) 963 } while (after.nonEmpty) 964 965 results.toList 966 } 967 968 def ping(): String = execute("conduit.ping").get_string 969 970 971 /* users */ 972 973 lazy val user_phid: String = execute("user.whoami").get_value(JSON.string(_, "phid")) 974 lazy val user_name: String = execute("user.whoami").get_value(JSON.string(_, "userName")) 975 976 def get_users( 977 all: Boolean = false, 978 phid: String = "", 979 name: String = ""): List[API.User] = 980 { 981 val constraints: JSON.Object.T = 982 (for { (key, value) <- List("phids" -> phid, "usernames" -> name) if value.nonEmpty } 983 yield (key, List(value))).toMap 984 985 execute_search("user.search", 986 JSON.Object("queryKey" -> (if (all) "all" else "active"), "constraints" -> constraints), 987 data => JSON.value(data, "fields", fields => 988 for { 989 id <- JSON.long(data, "id") 990 phid <- JSON.string(data, "phid") 991 name <- JSON.string(fields, "username") 992 real_name <- JSON.string0(fields, "realName") 993 roles <- JSON.strings(fields, "roles") 994 } yield API.User(id, phid, name, real_name, roles))) 995 } 996 997 def the_user(phid: String): API.User = 998 get_users(phid = phid) match { 999 case List(user) => user 1000 case _ => error("Bad user PHID " + quote(phid)) 1001 } 1002 1003 1004 /* repositories */ 1005 1006 def get_repositories( 1007 all: Boolean = false, 1008 phid: String = "", 1009 callsign: String = "", 1010 short_name: String = ""): List[API.Repository] = 1011 { 1012 val constraints: JSON.Object.T = 1013 (for { 1014 (key, value) <- List("phids" -> phid, "callsigns" -> callsign, "shortNames" -> short_name) 1015 if value.nonEmpty 1016 } yield (key, List(value))).toMap 1017 1018 execute_search("diffusion.repository.search", 1019 JSON.Object("queryKey" -> (if (all) "all" else "active"), "constraints" -> constraints), 1020 data => JSON.value(data, "fields", fields => 1021 for { 1022 vcs_name <- JSON.string(fields, "vcs") 1023 id <- JSON.long(data, "id") 1024 phid <- JSON.string(data, "phid") 1025 name <- JSON.string(fields, "name") 1026 callsign <- JSON.string0(fields, "callsign") 1027 short_name <- JSON.string0(fields, "shortName") 1028 importing <- JSON.bool(fields, "isImporting") 1029 } 1030 yield { 1031 val vcs = API.VCS.read(vcs_name) 1032 val url_path = 1033 if (short_name.isEmpty) "/diffusion/" + id else "/source/" + short_name 1034 val ssh_url = 1035 vcs match { 1036 case API.VCS.hg => hg_url + url_path 1037 case API.VCS.git => hg_url + url_path + ".git" 1038 case API.VCS.svn => "" 1039 } 1040 API.Repository(vcs, id, phid, name, callsign, short_name, importing, ssh_url) 1041 })) 1042 } 1043 1044 def the_repository(phid: String): API.Repository = 1045 get_repositories(phid = phid) match { 1046 case List(repo) => repo 1047 case _ => error("Bad repository PHID " + quote(phid)) 1048 } 1049 1050 def create_repository( 1051 name: String, 1052 callsign: String = "", // unique name, UPPERCASE 1053 short_name: String = "", // unique name 1054 description: String = "", 1055 public: Boolean = false, 1056 vcs: API.VCS.Value = API.VCS.hg): API.Repository = 1057 { 1058 require(name.nonEmpty) 1059 1060 val transactions = 1061 API.edits("vcs", vcs.toString) ::: 1062 API.edits("name", name) ::: 1063 API.opt_edits("callsign", proper_string(callsign)) ::: 1064 API.opt_edits("shortName", proper_string(short_name)) ::: 1065 API.opt_edits("description", proper_string(description)) ::: 1066 (if (public) Nil 1067 else API.edits("view", user_phid) ::: API.edits("policy.push", user_phid)) ::: 1068 API.edits("status", "active") 1069 1070 val phid = 1071 execute("diffusion.repository.edit", params = JSON.Object("transactions" -> transactions)) 1072 .get_value(JSON.value(_, "object", JSON.string(_, "phid"))) 1073 1074 execute("diffusion.looksoon", params = JSON.Object("repositories" -> List(phid))).get 1075 1076 the_repository(phid) 1077 } 1078 } 1079} 1080