hastd.c revision 218138
1/*- 2 * Copyright (c) 2009-2010 The FreeBSD Foundation 3 * Copyright (c) 2010-2011 Pawel Jakub Dawidek <pjd@FreeBSD.org> 4 * All rights reserved. 5 * 6 * This software was developed by Pawel Jakub Dawidek under sponsorship from 7 * the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD: head/sbin/hastd/hastd.c 218138 2011-01-31 18:32:17Z pjd $"); 33 34#include <sys/param.h> 35#include <sys/linker.h> 36#include <sys/module.h> 37#include <sys/stat.h> 38#include <sys/wait.h> 39 40#include <err.h> 41#include <errno.h> 42#include <libutil.h> 43#include <signal.h> 44#include <stdbool.h> 45#include <stdio.h> 46#include <stdlib.h> 47#include <string.h> 48#include <sysexits.h> 49#include <unistd.h> 50 51#include <activemap.h> 52#include <pjdlog.h> 53 54#include "control.h" 55#include "event.h" 56#include "hast.h" 57#include "hast_proto.h" 58#include "hastd.h" 59#include "hooks.h" 60#include "subr.h" 61 62/* Path to configuration file. */ 63const char *cfgpath = HAST_CONFIG; 64/* Hastd configuration. */ 65static struct hastd_config *cfg; 66/* Was SIGINT or SIGTERM signal received? */ 67bool sigexit_received = false; 68/* PID file handle. */ 69struct pidfh *pfh; 70 71/* How often check for hooks running for too long. */ 72#define REPORT_INTERVAL 5 73 74static void 75usage(void) 76{ 77 78 errx(EX_USAGE, "[-dFh] [-c config] [-P pidfile]"); 79} 80 81static void 82g_gate_load(void) 83{ 84 85 if (modfind("g_gate") == -1) { 86 /* Not present in kernel, try loading it. */ 87 if (kldload("geom_gate") == -1 || modfind("g_gate") == -1) { 88 if (errno != EEXIST) { 89 pjdlog_exit(EX_OSERR, 90 "Unable to load geom_gate module"); 91 } 92 } 93 } 94} 95 96void 97descriptors_cleanup(struct hast_resource *res) 98{ 99 struct hast_resource *tres; 100 101 TAILQ_FOREACH(tres, &cfg->hc_resources, hr_next) { 102 if (tres == res) { 103 PJDLOG_VERIFY(res->hr_role == HAST_ROLE_SECONDARY || 104 (res->hr_remotein == NULL && 105 res->hr_remoteout == NULL)); 106 continue; 107 } 108 if (tres->hr_remotein != NULL) 109 proto_close(tres->hr_remotein); 110 if (tres->hr_remoteout != NULL) 111 proto_close(tres->hr_remoteout); 112 } 113 if (cfg->hc_controlin != NULL) 114 proto_close(cfg->hc_controlin); 115 proto_close(cfg->hc_controlconn); 116 proto_close(cfg->hc_listenconn); 117 (void)pidfile_close(pfh); 118 hook_fini(); 119 pjdlog_fini(); 120} 121 122static const char * 123dtype2str(mode_t mode) 124{ 125 126 if (S_ISBLK(mode)) 127 return ("block device"); 128 else if (S_ISCHR(mode)) 129 return ("character device"); 130 else if (S_ISDIR(mode)) 131 return ("directory"); 132 else if (S_ISFIFO(mode)) 133 return ("pipe or FIFO"); 134 else if (S_ISLNK(mode)) 135 return ("symbolic link"); 136 else if (S_ISREG(mode)) 137 return ("regular file"); 138 else if (S_ISSOCK(mode)) 139 return ("socket"); 140 else if (S_ISWHT(mode)) 141 return ("whiteout"); 142 else 143 return ("unknown"); 144} 145 146void 147descriptors_assert(const struct hast_resource *res, int pjdlogmode) 148{ 149 char msg[256]; 150 struct stat sb; 151 long maxfd; 152 bool isopen; 153 mode_t mode; 154 int fd; 155 156 /* 157 * At this point descriptor to syslog socket is closed, so if we want 158 * to log assertion message, we have to first store it in 'msg' local 159 * buffer and then open syslog socket and log it. 160 */ 161 msg[0] = '\0'; 162 163 maxfd = sysconf(_SC_OPEN_MAX); 164 if (maxfd < 0) { 165 pjdlog_errno(LOG_WARNING, "sysconf(_SC_OPEN_MAX) failed"); 166 maxfd = 16384; 167 } 168 for (fd = 0; fd <= maxfd; fd++) { 169 if (fstat(fd, &sb) == 0) { 170 isopen = true; 171 mode = sb.st_mode; 172 } else if (errno == EBADF) { 173 isopen = false; 174 mode = 0; 175 } else { 176 isopen = true; /* silence gcc */ 177 mode = 0; /* silence gcc */ 178 snprintf(msg, sizeof(msg), 179 "Unable to fstat descriptor %d: %s", fd, 180 strerror(errno)); 181 } 182 if (fd == STDIN_FILENO || fd == STDOUT_FILENO || 183 fd == STDERR_FILENO) { 184 if (!isopen) { 185 snprintf(msg, sizeof(msg), 186 "Descriptor %d (%s) is closed, but should be open.", 187 fd, (fd == STDIN_FILENO ? "stdin" : 188 (fd == STDOUT_FILENO ? "stdout" : "stderr"))); 189 break; 190 } 191 } else if (fd == proto_descriptor(res->hr_event)) { 192 if (!isopen) { 193 snprintf(msg, sizeof(msg), 194 "Descriptor %d (event) is closed, but should be open.", 195 fd); 196 break; 197 } 198 if (!S_ISSOCK(mode)) { 199 snprintf(msg, sizeof(msg), 200 "Descriptor %d (event) is %s, but should be %s.", 201 fd, dtype2str(mode), dtype2str(S_IFSOCK)); 202 break; 203 } 204 } else if (fd == proto_descriptor(res->hr_ctrl)) { 205 if (!isopen) { 206 snprintf(msg, sizeof(msg), 207 "Descriptor %d (ctrl) is closed, but should be open.", 208 fd); 209 break; 210 } 211 if (!S_ISSOCK(mode)) { 212 snprintf(msg, sizeof(msg), 213 "Descriptor %d (ctrl) is %s, but should be %s.", 214 fd, dtype2str(mode), dtype2str(S_IFSOCK)); 215 break; 216 } 217 } else if (res->hr_role == HAST_ROLE_SECONDARY && 218 fd == proto_descriptor(res->hr_remotein)) { 219 if (!isopen) { 220 snprintf(msg, sizeof(msg), 221 "Descriptor %d (remote in) is closed, but should be open.", 222 fd); 223 break; 224 } 225 if (!S_ISSOCK(mode)) { 226 snprintf(msg, sizeof(msg), 227 "Descriptor %d (remote in) is %s, but should be %s.", 228 fd, dtype2str(mode), dtype2str(S_IFSOCK)); 229 break; 230 } 231 } else if (res->hr_role == HAST_ROLE_SECONDARY && 232 fd == proto_descriptor(res->hr_remoteout)) { 233 if (!isopen) { 234 snprintf(msg, sizeof(msg), 235 "Descriptor %d (remote out) is closed, but should be open.", 236 fd); 237 break; 238 } 239 if (!S_ISSOCK(mode)) { 240 snprintf(msg, sizeof(msg), 241 "Descriptor %d (remote out) is %s, but should be %s.", 242 fd, dtype2str(mode), dtype2str(S_IFSOCK)); 243 break; 244 } 245 } else { 246 if (isopen) { 247 snprintf(msg, sizeof(msg), 248 "Descriptor %d is open (%s), but should be closed.", 249 fd, dtype2str(mode)); 250 break; 251 } 252 } 253 } 254 if (msg[0] != '\0') { 255 pjdlog_init(pjdlogmode); 256 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, 257 role2str(res->hr_role)); 258 PJDLOG_ABORT("%s", msg); 259 } 260} 261 262static void 263child_exit_log(unsigned int pid, int status) 264{ 265 266 if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { 267 pjdlog_debug(1, "Worker process exited gracefully (pid=%u).", 268 pid); 269 } else if (WIFSIGNALED(status)) { 270 pjdlog_error("Worker process killed (pid=%u, signal=%d).", 271 pid, WTERMSIG(status)); 272 } else { 273 pjdlog_error("Worker process exited ungracefully (pid=%u, exitcode=%d).", 274 pid, WIFEXITED(status) ? WEXITSTATUS(status) : -1); 275 } 276} 277 278static void 279child_exit(void) 280{ 281 struct hast_resource *res; 282 int status; 283 pid_t pid; 284 285 while ((pid = wait3(&status, WNOHANG, NULL)) > 0) { 286 /* Find resource related to the process that just exited. */ 287 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 288 if (pid == res->hr_workerpid) 289 break; 290 } 291 if (res == NULL) { 292 /* 293 * This can happen when new connection arrives and we 294 * cancel child responsible for the old one or if this 295 * was hook which we executed. 296 */ 297 hook_check_one(pid, status); 298 continue; 299 } 300 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, 301 role2str(res->hr_role)); 302 child_exit_log(pid, status); 303 child_cleanup(res); 304 if (res->hr_role == HAST_ROLE_PRIMARY) { 305 /* 306 * Restart child process if it was killed by signal 307 * or exited because of temporary problem. 308 */ 309 if (WIFSIGNALED(status) || 310 (WIFEXITED(status) && 311 WEXITSTATUS(status) == EX_TEMPFAIL)) { 312 sleep(1); 313 pjdlog_info("Restarting worker process."); 314 hastd_primary(res); 315 } else { 316 res->hr_role = HAST_ROLE_INIT; 317 pjdlog_info("Changing resource role back to %s.", 318 role2str(res->hr_role)); 319 } 320 } 321 pjdlog_prefix_set("%s", ""); 322 } 323} 324 325static bool 326resource_needs_restart(const struct hast_resource *res0, 327 const struct hast_resource *res1) 328{ 329 330 PJDLOG_ASSERT(strcmp(res0->hr_name, res1->hr_name) == 0); 331 332 if (strcmp(res0->hr_provname, res1->hr_provname) != 0) 333 return (true); 334 if (strcmp(res0->hr_localpath, res1->hr_localpath) != 0) 335 return (true); 336 if (res0->hr_role == HAST_ROLE_INIT || 337 res0->hr_role == HAST_ROLE_SECONDARY) { 338 if (strcmp(res0->hr_remoteaddr, res1->hr_remoteaddr) != 0) 339 return (true); 340 if (res0->hr_replication != res1->hr_replication) 341 return (true); 342 if (res0->hr_timeout != res1->hr_timeout) 343 return (true); 344 if (strcmp(res0->hr_exec, res1->hr_exec) != 0) 345 return (true); 346 } 347 return (false); 348} 349 350static bool 351resource_needs_reload(const struct hast_resource *res0, 352 const struct hast_resource *res1) 353{ 354 355 PJDLOG_ASSERT(strcmp(res0->hr_name, res1->hr_name) == 0); 356 PJDLOG_ASSERT(strcmp(res0->hr_provname, res1->hr_provname) == 0); 357 PJDLOG_ASSERT(strcmp(res0->hr_localpath, res1->hr_localpath) == 0); 358 359 if (res0->hr_role != HAST_ROLE_PRIMARY) 360 return (false); 361 362 if (strcmp(res0->hr_remoteaddr, res1->hr_remoteaddr) != 0) 363 return (true); 364 if (res0->hr_replication != res1->hr_replication) 365 return (true); 366 if (res0->hr_timeout != res1->hr_timeout) 367 return (true); 368 if (strcmp(res0->hr_exec, res1->hr_exec) != 0) 369 return (true); 370 return (false); 371} 372 373static void 374resource_reload(const struct hast_resource *res) 375{ 376 struct nv *nvin, *nvout; 377 int error; 378 379 PJDLOG_ASSERT(res->hr_role == HAST_ROLE_PRIMARY); 380 381 nvout = nv_alloc(); 382 nv_add_uint8(nvout, HASTCTL_RELOAD, "cmd"); 383 nv_add_string(nvout, res->hr_remoteaddr, "remoteaddr"); 384 nv_add_int32(nvout, (int32_t)res->hr_replication, "replication"); 385 nv_add_int32(nvout, (int32_t)res->hr_timeout, "timeout"); 386 nv_add_string(nvout, res->hr_exec, "exec"); 387 if (nv_error(nvout) != 0) { 388 nv_free(nvout); 389 pjdlog_error("Unable to allocate header for reload message."); 390 return; 391 } 392 if (hast_proto_send(res, res->hr_ctrl, nvout, NULL, 0) < 0) { 393 pjdlog_errno(LOG_ERR, "Unable to send reload message"); 394 nv_free(nvout); 395 return; 396 } 397 nv_free(nvout); 398 399 /* Receive response. */ 400 if (hast_proto_recv_hdr(res->hr_ctrl, &nvin) < 0) { 401 pjdlog_errno(LOG_ERR, "Unable to receive reload reply"); 402 return; 403 } 404 error = nv_get_int16(nvin, "error"); 405 nv_free(nvin); 406 if (error != 0) { 407 pjdlog_common(LOG_ERR, 0, error, "Reload failed"); 408 return; 409 } 410} 411 412static void 413hastd_reload(void) 414{ 415 struct hastd_config *newcfg; 416 struct hast_resource *nres, *cres, *tres; 417 uint8_t role; 418 419 pjdlog_info("Reloading configuration..."); 420 421 newcfg = yy_config_parse(cfgpath, false); 422 if (newcfg == NULL) 423 goto failed; 424 425 /* 426 * Check if control address has changed. 427 */ 428 if (strcmp(cfg->hc_controladdr, newcfg->hc_controladdr) != 0) { 429 if (proto_server(newcfg->hc_controladdr, 430 &newcfg->hc_controlconn) < 0) { 431 pjdlog_errno(LOG_ERR, 432 "Unable to listen on control address %s", 433 newcfg->hc_controladdr); 434 goto failed; 435 } 436 } 437 /* 438 * Check if listen address has changed. 439 */ 440 if (strcmp(cfg->hc_listenaddr, newcfg->hc_listenaddr) != 0) { 441 if (proto_server(newcfg->hc_listenaddr, 442 &newcfg->hc_listenconn) < 0) { 443 pjdlog_errno(LOG_ERR, "Unable to listen on address %s", 444 newcfg->hc_listenaddr); 445 goto failed; 446 } 447 } 448 /* 449 * Only when both control and listen sockets are successfully 450 * initialized switch them to new configuration. 451 */ 452 if (newcfg->hc_controlconn != NULL) { 453 pjdlog_info("Control socket changed from %s to %s.", 454 cfg->hc_controladdr, newcfg->hc_controladdr); 455 proto_close(cfg->hc_controlconn); 456 cfg->hc_controlconn = newcfg->hc_controlconn; 457 newcfg->hc_controlconn = NULL; 458 strlcpy(cfg->hc_controladdr, newcfg->hc_controladdr, 459 sizeof(cfg->hc_controladdr)); 460 } 461 if (newcfg->hc_listenconn != NULL) { 462 pjdlog_info("Listen socket changed from %s to %s.", 463 cfg->hc_listenaddr, newcfg->hc_listenaddr); 464 proto_close(cfg->hc_listenconn); 465 cfg->hc_listenconn = newcfg->hc_listenconn; 466 newcfg->hc_listenconn = NULL; 467 strlcpy(cfg->hc_listenaddr, newcfg->hc_listenaddr, 468 sizeof(cfg->hc_listenaddr)); 469 } 470 471 /* 472 * Stop and remove resources that were removed from the configuration. 473 */ 474 TAILQ_FOREACH_SAFE(cres, &cfg->hc_resources, hr_next, tres) { 475 TAILQ_FOREACH(nres, &newcfg->hc_resources, hr_next) { 476 if (strcmp(cres->hr_name, nres->hr_name) == 0) 477 break; 478 } 479 if (nres == NULL) { 480 control_set_role(cres, HAST_ROLE_INIT); 481 TAILQ_REMOVE(&cfg->hc_resources, cres, hr_next); 482 pjdlog_info("Resource %s removed.", cres->hr_name); 483 free(cres); 484 } 485 } 486 /* 487 * Move new resources to the current configuration. 488 */ 489 TAILQ_FOREACH_SAFE(nres, &newcfg->hc_resources, hr_next, tres) { 490 TAILQ_FOREACH(cres, &cfg->hc_resources, hr_next) { 491 if (strcmp(cres->hr_name, nres->hr_name) == 0) 492 break; 493 } 494 if (cres == NULL) { 495 TAILQ_REMOVE(&newcfg->hc_resources, nres, hr_next); 496 TAILQ_INSERT_TAIL(&cfg->hc_resources, nres, hr_next); 497 pjdlog_info("Resource %s added.", nres->hr_name); 498 } 499 } 500 /* 501 * Deal with modified resources. 502 * Depending on what has changed exactly we might want to perform 503 * different actions. 504 * 505 * We do full resource restart in the following situations: 506 * Resource role is INIT or SECONDARY. 507 * Resource role is PRIMARY and path to local component or provider 508 * name has changed. 509 * In case of PRIMARY, the worker process will be killed and restarted, 510 * which also means removing /dev/hast/<name> provider and 511 * recreating it. 512 * 513 * We do just reload (send SIGHUP to worker process) if we act as 514 * PRIMARY, but only if remote address, replication mode, timeout or 515 * execution path has changed. For those, there is no need to restart 516 * worker process. 517 * If PRIMARY receives SIGHUP, it will reconnect if remote address or 518 * replication mode has changed or simply set new timeout if only 519 * timeout has changed. 520 */ 521 TAILQ_FOREACH_SAFE(nres, &newcfg->hc_resources, hr_next, tres) { 522 TAILQ_FOREACH(cres, &cfg->hc_resources, hr_next) { 523 if (strcmp(cres->hr_name, nres->hr_name) == 0) 524 break; 525 } 526 PJDLOG_ASSERT(cres != NULL); 527 if (resource_needs_restart(cres, nres)) { 528 pjdlog_info("Resource %s configuration was modified, restarting it.", 529 cres->hr_name); 530 role = cres->hr_role; 531 control_set_role(cres, HAST_ROLE_INIT); 532 TAILQ_REMOVE(&cfg->hc_resources, cres, hr_next); 533 free(cres); 534 TAILQ_REMOVE(&newcfg->hc_resources, nres, hr_next); 535 TAILQ_INSERT_TAIL(&cfg->hc_resources, nres, hr_next); 536 control_set_role(nres, role); 537 } else if (resource_needs_reload(cres, nres)) { 538 pjdlog_info("Resource %s configuration was modified, reloading it.", 539 cres->hr_name); 540 strlcpy(cres->hr_remoteaddr, nres->hr_remoteaddr, 541 sizeof(cres->hr_remoteaddr)); 542 cres->hr_replication = nres->hr_replication; 543 cres->hr_timeout = nres->hr_timeout; 544 strlcpy(cres->hr_exec, nres->hr_exec, 545 sizeof(cres->hr_exec)); 546 if (cres->hr_workerpid != 0) 547 resource_reload(cres); 548 } 549 } 550 551 yy_config_free(newcfg); 552 pjdlog_info("Configuration reloaded successfully."); 553 return; 554failed: 555 if (newcfg != NULL) { 556 if (newcfg->hc_controlconn != NULL) 557 proto_close(newcfg->hc_controlconn); 558 if (newcfg->hc_listenconn != NULL) 559 proto_close(newcfg->hc_listenconn); 560 yy_config_free(newcfg); 561 } 562 pjdlog_warning("Configuration not reloaded."); 563} 564 565static void 566terminate_workers(void) 567{ 568 struct hast_resource *res; 569 570 pjdlog_info("Termination signal received, exiting."); 571 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 572 if (res->hr_workerpid == 0) 573 continue; 574 pjdlog_info("Terminating worker process (resource=%s, role=%s, pid=%u).", 575 res->hr_name, role2str(res->hr_role), res->hr_workerpid); 576 if (kill(res->hr_workerpid, SIGTERM) == 0) 577 continue; 578 pjdlog_errno(LOG_WARNING, 579 "Unable to send signal to worker process (resource=%s, role=%s, pid=%u).", 580 res->hr_name, role2str(res->hr_role), res->hr_workerpid); 581 } 582} 583 584static void 585listen_accept(void) 586{ 587 struct hast_resource *res; 588 struct proto_conn *conn; 589 struct nv *nvin, *nvout, *nverr; 590 const char *resname; 591 const unsigned char *token; 592 char laddr[256], raddr[256]; 593 size_t size; 594 pid_t pid; 595 int status; 596 597 proto_local_address(cfg->hc_listenconn, laddr, sizeof(laddr)); 598 pjdlog_debug(1, "Accepting connection to %s.", laddr); 599 600 if (proto_accept(cfg->hc_listenconn, &conn) < 0) { 601 pjdlog_errno(LOG_ERR, "Unable to accept connection %s", laddr); 602 return; 603 } 604 605 proto_local_address(conn, laddr, sizeof(laddr)); 606 proto_remote_address(conn, raddr, sizeof(raddr)); 607 pjdlog_info("Connection from %s to %s.", raddr, laddr); 608 609 /* Error in setting timeout is not critical, but why should it fail? */ 610 if (proto_timeout(conn, HAST_TIMEOUT) < 0) 611 pjdlog_errno(LOG_WARNING, "Unable to set connection timeout"); 612 613 nvin = nvout = nverr = NULL; 614 615 /* 616 * Before receiving any data see if remote host have access to any 617 * resource. 618 */ 619 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 620 if (proto_address_match(conn, res->hr_remoteaddr)) 621 break; 622 } 623 if (res == NULL) { 624 pjdlog_error("Client %s isn't known.", raddr); 625 goto close; 626 } 627 /* Ok, remote host can access at least one resource. */ 628 629 if (hast_proto_recv_hdr(conn, &nvin) < 0) { 630 pjdlog_errno(LOG_ERR, "Unable to receive header from %s", 631 raddr); 632 goto close; 633 } 634 635 resname = nv_get_string(nvin, "resource"); 636 if (resname == NULL) { 637 pjdlog_error("No 'resource' field in the header received from %s.", 638 raddr); 639 goto close; 640 } 641 pjdlog_debug(2, "%s: resource=%s", raddr, resname); 642 token = nv_get_uint8_array(nvin, &size, "token"); 643 /* 644 * NULL token means that this is first conection. 645 */ 646 if (token != NULL && size != sizeof(res->hr_token)) { 647 pjdlog_error("Received token of invalid size from %s (expected %zu, got %zu).", 648 raddr, sizeof(res->hr_token), size); 649 goto close; 650 } 651 652 /* 653 * From now on we want to send errors to the remote node. 654 */ 655 nverr = nv_alloc(); 656 657 /* Find resource related to this connection. */ 658 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 659 if (strcmp(resname, res->hr_name) == 0) 660 break; 661 } 662 /* Have we found the resource? */ 663 if (res == NULL) { 664 pjdlog_error("No resource '%s' as requested by %s.", 665 resname, raddr); 666 nv_add_stringf(nverr, "errmsg", "Resource not configured."); 667 goto fail; 668 } 669 670 /* Now that we know resource name setup log prefix. */ 671 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role)); 672 673 /* Does the remote host have access to this resource? */ 674 if (!proto_address_match(conn, res->hr_remoteaddr)) { 675 pjdlog_error("Client %s has no access to the resource.", raddr); 676 nv_add_stringf(nverr, "errmsg", "No access to the resource."); 677 goto fail; 678 } 679 /* Is the resource marked as secondary? */ 680 if (res->hr_role != HAST_ROLE_SECONDARY) { 681 pjdlog_error("We act as %s for the resource and not as %s as requested by %s.", 682 role2str(res->hr_role), role2str(HAST_ROLE_SECONDARY), 683 raddr); 684 nv_add_stringf(nverr, "errmsg", 685 "Remote node acts as %s for the resource and not as %s.", 686 role2str(res->hr_role), role2str(HAST_ROLE_SECONDARY)); 687 goto fail; 688 } 689 /* Does token (if exists) match? */ 690 if (token != NULL && memcmp(token, res->hr_token, 691 sizeof(res->hr_token)) != 0) { 692 pjdlog_error("Token received from %s doesn't match.", raddr); 693 nv_add_stringf(nverr, "errmsg", "Token doesn't match."); 694 goto fail; 695 } 696 /* 697 * If there is no token, but we have half-open connection 698 * (only remotein) or full connection (worker process is running) 699 * we have to cancel those and accept the new connection. 700 */ 701 if (token == NULL) { 702 PJDLOG_ASSERT(res->hr_remoteout == NULL); 703 pjdlog_debug(1, "Initial connection from %s.", raddr); 704 if (res->hr_workerpid != 0) { 705 PJDLOG_ASSERT(res->hr_remotein == NULL); 706 pjdlog_debug(1, 707 "Worker process exists (pid=%u), stopping it.", 708 (unsigned int)res->hr_workerpid); 709 /* Stop child process. */ 710 if (kill(res->hr_workerpid, SIGINT) < 0) { 711 pjdlog_errno(LOG_ERR, 712 "Unable to stop worker process (pid=%u)", 713 (unsigned int)res->hr_workerpid); 714 /* 715 * Other than logging the problem we 716 * ignore it - nothing smart to do. 717 */ 718 } 719 /* Wait for it to exit. */ 720 else if ((pid = waitpid(res->hr_workerpid, 721 &status, 0)) != res->hr_workerpid) { 722 /* We can only log the problem. */ 723 pjdlog_errno(LOG_ERR, 724 "Waiting for worker process (pid=%u) failed", 725 (unsigned int)res->hr_workerpid); 726 } else { 727 child_exit_log(res->hr_workerpid, status); 728 } 729 child_cleanup(res); 730 } else if (res->hr_remotein != NULL) { 731 char oaddr[256]; 732 733 proto_remote_address(res->hr_remotein, oaddr, 734 sizeof(oaddr)); 735 pjdlog_debug(1, 736 "Canceling half-open connection from %s on connection from %s.", 737 oaddr, raddr); 738 proto_close(res->hr_remotein); 739 res->hr_remotein = NULL; 740 } 741 } 742 743 /* 744 * Checks and cleanups are done. 745 */ 746 747 if (token == NULL) { 748 arc4random_buf(res->hr_token, sizeof(res->hr_token)); 749 nvout = nv_alloc(); 750 nv_add_uint8_array(nvout, res->hr_token, 751 sizeof(res->hr_token), "token"); 752 if (nv_error(nvout) != 0) { 753 pjdlog_common(LOG_ERR, 0, nv_error(nvout), 754 "Unable to prepare return header for %s", raddr); 755 nv_add_stringf(nverr, "errmsg", 756 "Remote node was unable to prepare return header: %s.", 757 strerror(nv_error(nvout))); 758 goto fail; 759 } 760 if (hast_proto_send(NULL, conn, nvout, NULL, 0) < 0) { 761 int error = errno; 762 763 pjdlog_errno(LOG_ERR, "Unable to send response to %s", 764 raddr); 765 nv_add_stringf(nverr, "errmsg", 766 "Remote node was unable to send response: %s.", 767 strerror(error)); 768 goto fail; 769 } 770 res->hr_remotein = conn; 771 pjdlog_debug(1, "Incoming connection from %s configured.", 772 raddr); 773 } else { 774 res->hr_remoteout = conn; 775 pjdlog_debug(1, "Outgoing connection to %s configured.", raddr); 776 hastd_secondary(res, nvin); 777 } 778 nv_free(nvin); 779 nv_free(nvout); 780 nv_free(nverr); 781 pjdlog_prefix_set("%s", ""); 782 return; 783fail: 784 if (nv_error(nverr) != 0) { 785 pjdlog_common(LOG_ERR, 0, nv_error(nverr), 786 "Unable to prepare error header for %s", raddr); 787 goto close; 788 } 789 if (hast_proto_send(NULL, conn, nverr, NULL, 0) < 0) { 790 pjdlog_errno(LOG_ERR, "Unable to send error to %s", raddr); 791 goto close; 792 } 793close: 794 if (nvin != NULL) 795 nv_free(nvin); 796 if (nvout != NULL) 797 nv_free(nvout); 798 if (nverr != NULL) 799 nv_free(nverr); 800 proto_close(conn); 801 pjdlog_prefix_set("%s", ""); 802} 803 804static void 805main_loop(void) 806{ 807 struct hast_resource *res; 808 struct timeval seltimeout; 809 struct timespec sigtimeout; 810 int fd, maxfd, ret, signo; 811 sigset_t mask; 812 fd_set rfds; 813 814 seltimeout.tv_sec = REPORT_INTERVAL; 815 seltimeout.tv_usec = 0; 816 sigtimeout.tv_sec = 0; 817 sigtimeout.tv_nsec = 0; 818 819 PJDLOG_VERIFY(sigemptyset(&mask) == 0); 820 PJDLOG_VERIFY(sigaddset(&mask, SIGHUP) == 0); 821 PJDLOG_VERIFY(sigaddset(&mask, SIGINT) == 0); 822 PJDLOG_VERIFY(sigaddset(&mask, SIGTERM) == 0); 823 PJDLOG_VERIFY(sigaddset(&mask, SIGCHLD) == 0); 824 825 pjdlog_info("Started successfully, running protocol version %d.", 826 HAST_PROTO_VERSION); 827 828 for (;;) { 829 while ((signo = sigtimedwait(&mask, NULL, &sigtimeout)) != -1) { 830 switch (signo) { 831 case SIGINT: 832 case SIGTERM: 833 sigexit_received = true; 834 terminate_workers(); 835 proto_close(cfg->hc_controlconn); 836 exit(EX_OK); 837 break; 838 case SIGCHLD: 839 child_exit(); 840 break; 841 case SIGHUP: 842 hastd_reload(); 843 break; 844 default: 845 PJDLOG_ABORT("Unexpected signal (%d).", signo); 846 } 847 } 848 849 /* Setup descriptors for select(2). */ 850 FD_ZERO(&rfds); 851 maxfd = fd = proto_descriptor(cfg->hc_controlconn); 852 PJDLOG_ASSERT(fd >= 0); 853 FD_SET(fd, &rfds); 854 fd = proto_descriptor(cfg->hc_listenconn); 855 PJDLOG_ASSERT(fd >= 0); 856 FD_SET(fd, &rfds); 857 maxfd = fd > maxfd ? fd : maxfd; 858 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 859 if (res->hr_event == NULL) 860 continue; 861 fd = proto_descriptor(res->hr_event); 862 PJDLOG_ASSERT(fd >= 0); 863 FD_SET(fd, &rfds); 864 maxfd = fd > maxfd ? fd : maxfd; 865 } 866 867 PJDLOG_ASSERT(maxfd + 1 <= (int)FD_SETSIZE); 868 ret = select(maxfd + 1, &rfds, NULL, NULL, &seltimeout); 869 if (ret == 0) 870 hook_check(); 871 else if (ret == -1) { 872 if (errno == EINTR) 873 continue; 874 KEEP_ERRNO((void)pidfile_remove(pfh)); 875 pjdlog_exit(EX_OSERR, "select() failed"); 876 } 877 878 if (FD_ISSET(proto_descriptor(cfg->hc_controlconn), &rfds)) 879 control_handle(cfg); 880 if (FD_ISSET(proto_descriptor(cfg->hc_listenconn), &rfds)) 881 listen_accept(); 882 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 883 if (res->hr_event == NULL) 884 continue; 885 if (FD_ISSET(proto_descriptor(res->hr_event), &rfds)) { 886 if (event_recv(res) == 0) 887 continue; 888 /* The worker process exited? */ 889 proto_close(res->hr_event); 890 res->hr_event = NULL; 891 } 892 } 893 } 894} 895 896static void 897dummy_sighandler(int sig __unused) 898{ 899 /* Nothing to do. */ 900} 901 902int 903main(int argc, char *argv[]) 904{ 905 const char *pidfile; 906 pid_t otherpid; 907 bool foreground; 908 int debuglevel; 909 sigset_t mask; 910 911 foreground = false; 912 debuglevel = 0; 913 pidfile = HASTD_PIDFILE; 914 915 for (;;) { 916 int ch; 917 918 ch = getopt(argc, argv, "c:dFhP:"); 919 if (ch == -1) 920 break; 921 switch (ch) { 922 case 'c': 923 cfgpath = optarg; 924 break; 925 case 'd': 926 debuglevel++; 927 break; 928 case 'F': 929 foreground = true; 930 break; 931 case 'P': 932 pidfile = optarg; 933 break; 934 case 'h': 935 default: 936 usage(); 937 } 938 } 939 argc -= optind; 940 argv += optind; 941 942 pjdlog_init(PJDLOG_MODE_STD); 943 pjdlog_debug_set(debuglevel); 944 945 g_gate_load(); 946 947 pfh = pidfile_open(pidfile, 0600, &otherpid); 948 if (pfh == NULL) { 949 if (errno == EEXIST) { 950 pjdlog_exitx(EX_TEMPFAIL, 951 "Another hastd is already running, pid: %jd.", 952 (intmax_t)otherpid); 953 } 954 /* If we cannot create pidfile from other reasons, only warn. */ 955 pjdlog_errno(LOG_WARNING, "Unable to open or create pidfile"); 956 } 957 958 cfg = yy_config_parse(cfgpath, true); 959 PJDLOG_ASSERT(cfg != NULL); 960 961 /* 962 * Restore default actions for interesting signals in case parent 963 * process (like init(8)) decided to ignore some of them (like SIGHUP). 964 */ 965 PJDLOG_VERIFY(signal(SIGHUP, SIG_DFL) != SIG_ERR); 966 PJDLOG_VERIFY(signal(SIGINT, SIG_DFL) != SIG_ERR); 967 PJDLOG_VERIFY(signal(SIGTERM, SIG_DFL) != SIG_ERR); 968 /* 969 * Because SIGCHLD is ignored by default, setup dummy handler for it, 970 * so we can mask it. 971 */ 972 PJDLOG_VERIFY(signal(SIGCHLD, dummy_sighandler) != SIG_ERR); 973 974 PJDLOG_VERIFY(sigemptyset(&mask) == 0); 975 PJDLOG_VERIFY(sigaddset(&mask, SIGHUP) == 0); 976 PJDLOG_VERIFY(sigaddset(&mask, SIGINT) == 0); 977 PJDLOG_VERIFY(sigaddset(&mask, SIGTERM) == 0); 978 PJDLOG_VERIFY(sigaddset(&mask, SIGCHLD) == 0); 979 PJDLOG_VERIFY(sigprocmask(SIG_SETMASK, &mask, NULL) == 0); 980 981 /* Listen on control address. */ 982 if (proto_server(cfg->hc_controladdr, &cfg->hc_controlconn) < 0) { 983 KEEP_ERRNO((void)pidfile_remove(pfh)); 984 pjdlog_exit(EX_OSERR, "Unable to listen on control address %s", 985 cfg->hc_controladdr); 986 } 987 /* Listen for remote connections. */ 988 if (proto_server(cfg->hc_listenaddr, &cfg->hc_listenconn) < 0) { 989 KEEP_ERRNO((void)pidfile_remove(pfh)); 990 pjdlog_exit(EX_OSERR, "Unable to listen on address %s", 991 cfg->hc_listenaddr); 992 } 993 994 if (!foreground) { 995 if (daemon(0, 0) < 0) { 996 KEEP_ERRNO((void)pidfile_remove(pfh)); 997 pjdlog_exit(EX_OSERR, "Unable to daemonize"); 998 } 999 1000 /* Start logging to syslog. */ 1001 pjdlog_mode_set(PJDLOG_MODE_SYSLOG); 1002 1003 /* Write PID to a file. */ 1004 if (pidfile_write(pfh) < 0) { 1005 pjdlog_errno(LOG_WARNING, 1006 "Unable to write PID to a file"); 1007 } 1008 } 1009 1010 hook_init(); 1011 1012 main_loop(); 1013 1014 exit(0); 1015} 1016