hastd.c revision 217729
1204076Spjd/*- 2204076Spjd * Copyright (c) 2009-2010 The FreeBSD Foundation 3210886Spjd * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org> 4204076Spjd * All rights reserved. 5204076Spjd * 6204076Spjd * This software was developed by Pawel Jakub Dawidek under sponsorship from 7204076Spjd * the FreeBSD Foundation. 8204076Spjd * 9204076Spjd * Redistribution and use in source and binary forms, with or without 10204076Spjd * modification, are permitted provided that the following conditions 11204076Spjd * are met: 12204076Spjd * 1. Redistributions of source code must retain the above copyright 13204076Spjd * notice, this list of conditions and the following disclaimer. 14204076Spjd * 2. Redistributions in binary form must reproduce the above copyright 15204076Spjd * notice, this list of conditions and the following disclaimer in the 16204076Spjd * documentation and/or other materials provided with the distribution. 17204076Spjd * 18204076Spjd * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 19204076Spjd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20204076Spjd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21204076Spjd * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 22204076Spjd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23204076Spjd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24204076Spjd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25204076Spjd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26204076Spjd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27204076Spjd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28204076Spjd * SUCH DAMAGE. 29204076Spjd */ 30204076Spjd 31204076Spjd#include <sys/cdefs.h> 32204076Spjd__FBSDID("$FreeBSD: head/sbin/hastd/hastd.c 217729 2011-01-22 22:31:55Z pjd $"); 33204076Spjd 34204076Spjd#include <sys/param.h> 35204076Spjd#include <sys/linker.h> 36204076Spjd#include <sys/module.h> 37204076Spjd#include <sys/wait.h> 38204076Spjd 39204076Spjd#include <assert.h> 40204076Spjd#include <err.h> 41204076Spjd#include <errno.h> 42204076Spjd#include <libutil.h> 43204076Spjd#include <signal.h> 44204076Spjd#include <stdbool.h> 45204076Spjd#include <stdio.h> 46204076Spjd#include <stdlib.h> 47204076Spjd#include <string.h> 48204076Spjd#include <sysexits.h> 49204076Spjd#include <unistd.h> 50204076Spjd 51204076Spjd#include <activemap.h> 52204076Spjd#include <pjdlog.h> 53204076Spjd 54204076Spjd#include "control.h" 55212038Spjd#include "event.h" 56204076Spjd#include "hast.h" 57204076Spjd#include "hast_proto.h" 58204076Spjd#include "hastd.h" 59211977Spjd#include "hooks.h" 60204076Spjd#include "subr.h" 61204076Spjd 62204076Spjd/* Path to configuration file. */ 63210886Spjdconst char *cfgpath = HAST_CONFIG; 64204076Spjd/* Hastd configuration. */ 65204076Spjdstatic struct hastd_config *cfg; 66204076Spjd/* Was SIGINT or SIGTERM signal received? */ 67204076Spjdbool sigexit_received = false; 68204076Spjd/* PID file handle. */ 69204076Spjdstruct pidfh *pfh; 70204076Spjd 71211977Spjd/* How often check for hooks running for too long. */ 72213430Spjd#define REPORT_INTERVAL 5 73211977Spjd 74204076Spjdstatic void 75204076Spjdusage(void) 76204076Spjd{ 77204076Spjd 78204076Spjd errx(EX_USAGE, "[-dFh] [-c config] [-P pidfile]"); 79204076Spjd} 80204076Spjd 81204076Spjdstatic void 82204076Spjdg_gate_load(void) 83204076Spjd{ 84204076Spjd 85204076Spjd if (modfind("g_gate") == -1) { 86204076Spjd /* Not present in kernel, try loading it. */ 87204076Spjd if (kldload("geom_gate") == -1 || modfind("g_gate") == -1) { 88204076Spjd if (errno != EEXIST) { 89204076Spjd pjdlog_exit(EX_OSERR, 90204076Spjd "Unable to load geom_gate module"); 91204076Spjd } 92204076Spjd } 93204076Spjd } 94204076Spjd} 95204076Spjd 96204076Spjdstatic void 97207372Spjdchild_exit_log(unsigned int pid, int status) 98207372Spjd{ 99207372Spjd 100207372Spjd if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { 101207372Spjd pjdlog_debug(1, "Worker process exited gracefully (pid=%u).", 102207372Spjd pid); 103207372Spjd } else if (WIFSIGNALED(status)) { 104207372Spjd pjdlog_error("Worker process killed (pid=%u, signal=%d).", 105207372Spjd pid, WTERMSIG(status)); 106207372Spjd } else { 107207372Spjd pjdlog_error("Worker process exited ungracefully (pid=%u, exitcode=%d).", 108207372Spjd pid, WIFEXITED(status) ? WEXITSTATUS(status) : -1); 109207372Spjd } 110207372Spjd} 111207372Spjd 112207372Spjdstatic void 113204076Spjdchild_exit(void) 114204076Spjd{ 115204076Spjd struct hast_resource *res; 116204076Spjd int status; 117204076Spjd pid_t pid; 118204076Spjd 119204076Spjd while ((pid = wait3(&status, WNOHANG, NULL)) > 0) { 120204076Spjd /* Find resource related to the process that just exited. */ 121204076Spjd TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 122204076Spjd if (pid == res->hr_workerpid) 123204076Spjd break; 124204076Spjd } 125204076Spjd if (res == NULL) { 126204076Spjd /* 127204076Spjd * This can happen when new connection arrives and we 128211977Spjd * cancel child responsible for the old one or if this 129211977Spjd * was hook which we executed. 130204076Spjd */ 131211977Spjd hook_check_one(pid, status); 132204076Spjd continue; 133204076Spjd } 134204076Spjd pjdlog_prefix_set("[%s] (%s) ", res->hr_name, 135204076Spjd role2str(res->hr_role)); 136207372Spjd child_exit_log(pid, status); 137213006Spjd child_cleanup(res); 138204076Spjd if (res->hr_role == HAST_ROLE_PRIMARY) { 139207372Spjd /* 140207372Spjd * Restart child process if it was killed by signal 141207372Spjd * or exited because of temporary problem. 142207372Spjd */ 143207372Spjd if (WIFSIGNALED(status) || 144207372Spjd (WIFEXITED(status) && 145207372Spjd WEXITSTATUS(status) == EX_TEMPFAIL)) { 146207348Spjd sleep(1); 147207348Spjd pjdlog_info("Restarting worker process."); 148207348Spjd hastd_primary(res); 149207348Spjd } else { 150207348Spjd res->hr_role = HAST_ROLE_INIT; 151207348Spjd pjdlog_info("Changing resource role back to %s.", 152207348Spjd role2str(res->hr_role)); 153207348Spjd } 154204076Spjd } 155204076Spjd pjdlog_prefix_set("%s", ""); 156204076Spjd } 157204076Spjd} 158204076Spjd 159210886Spjdstatic bool 160210886Spjdresource_needs_restart(const struct hast_resource *res0, 161210886Spjd const struct hast_resource *res1) 162210886Spjd{ 163210886Spjd 164210886Spjd assert(strcmp(res0->hr_name, res1->hr_name) == 0); 165210886Spjd 166210886Spjd if (strcmp(res0->hr_provname, res1->hr_provname) != 0) 167210886Spjd return (true); 168210886Spjd if (strcmp(res0->hr_localpath, res1->hr_localpath) != 0) 169210886Spjd return (true); 170210886Spjd if (res0->hr_role == HAST_ROLE_INIT || 171210886Spjd res0->hr_role == HAST_ROLE_SECONDARY) { 172210886Spjd if (strcmp(res0->hr_remoteaddr, res1->hr_remoteaddr) != 0) 173210886Spjd return (true); 174210886Spjd if (res0->hr_replication != res1->hr_replication) 175210886Spjd return (true); 176210886Spjd if (res0->hr_timeout != res1->hr_timeout) 177210886Spjd return (true); 178211886Spjd if (strcmp(res0->hr_exec, res1->hr_exec) != 0) 179211886Spjd return (true); 180210886Spjd } 181210886Spjd return (false); 182210886Spjd} 183210886Spjd 184210886Spjdstatic bool 185210886Spjdresource_needs_reload(const struct hast_resource *res0, 186210886Spjd const struct hast_resource *res1) 187210886Spjd{ 188210886Spjd 189210886Spjd assert(strcmp(res0->hr_name, res1->hr_name) == 0); 190210886Spjd assert(strcmp(res0->hr_provname, res1->hr_provname) == 0); 191210886Spjd assert(strcmp(res0->hr_localpath, res1->hr_localpath) == 0); 192210886Spjd 193210886Spjd if (res0->hr_role != HAST_ROLE_PRIMARY) 194210886Spjd return (false); 195210886Spjd 196210886Spjd if (strcmp(res0->hr_remoteaddr, res1->hr_remoteaddr) != 0) 197210886Spjd return (true); 198210886Spjd if (res0->hr_replication != res1->hr_replication) 199210886Spjd return (true); 200210886Spjd if (res0->hr_timeout != res1->hr_timeout) 201210886Spjd return (true); 202211886Spjd if (strcmp(res0->hr_exec, res1->hr_exec) != 0) 203211886Spjd return (true); 204210886Spjd return (false); 205210886Spjd} 206210886Spjd 207204076Spjdstatic void 208204076Spjdhastd_reload(void) 209204076Spjd{ 210210886Spjd struct hastd_config *newcfg; 211210886Spjd struct hast_resource *nres, *cres, *tres; 212210886Spjd uint8_t role; 213204076Spjd 214210886Spjd pjdlog_info("Reloading configuration..."); 215210886Spjd 216210886Spjd newcfg = yy_config_parse(cfgpath, false); 217210886Spjd if (newcfg == NULL) 218210886Spjd goto failed; 219210886Spjd 220210886Spjd /* 221210886Spjd * Check if control address has changed. 222210886Spjd */ 223210886Spjd if (strcmp(cfg->hc_controladdr, newcfg->hc_controladdr) != 0) { 224210886Spjd if (proto_server(newcfg->hc_controladdr, 225210886Spjd &newcfg->hc_controlconn) < 0) { 226210886Spjd pjdlog_errno(LOG_ERR, 227210886Spjd "Unable to listen on control address %s", 228210886Spjd newcfg->hc_controladdr); 229210886Spjd goto failed; 230210886Spjd } 231210886Spjd } 232210886Spjd /* 233210886Spjd * Check if listen address has changed. 234210886Spjd */ 235210886Spjd if (strcmp(cfg->hc_listenaddr, newcfg->hc_listenaddr) != 0) { 236210886Spjd if (proto_server(newcfg->hc_listenaddr, 237210886Spjd &newcfg->hc_listenconn) < 0) { 238210886Spjd pjdlog_errno(LOG_ERR, "Unable to listen on address %s", 239210886Spjd newcfg->hc_listenaddr); 240210886Spjd goto failed; 241210886Spjd } 242210886Spjd } 243210886Spjd /* 244210886Spjd * Only when both control and listen sockets are successfully 245210886Spjd * initialized switch them to new configuration. 246210886Spjd */ 247210886Spjd if (newcfg->hc_controlconn != NULL) { 248210886Spjd pjdlog_info("Control socket changed from %s to %s.", 249210886Spjd cfg->hc_controladdr, newcfg->hc_controladdr); 250210886Spjd proto_close(cfg->hc_controlconn); 251210886Spjd cfg->hc_controlconn = newcfg->hc_controlconn; 252210886Spjd newcfg->hc_controlconn = NULL; 253210886Spjd strlcpy(cfg->hc_controladdr, newcfg->hc_controladdr, 254210886Spjd sizeof(cfg->hc_controladdr)); 255210886Spjd } 256210886Spjd if (newcfg->hc_listenconn != NULL) { 257210886Spjd pjdlog_info("Listen socket changed from %s to %s.", 258210886Spjd cfg->hc_listenaddr, newcfg->hc_listenaddr); 259210886Spjd proto_close(cfg->hc_listenconn); 260210886Spjd cfg->hc_listenconn = newcfg->hc_listenconn; 261210886Spjd newcfg->hc_listenconn = NULL; 262210886Spjd strlcpy(cfg->hc_listenaddr, newcfg->hc_listenaddr, 263210886Spjd sizeof(cfg->hc_listenaddr)); 264210886Spjd } 265210886Spjd 266210886Spjd /* 267210886Spjd * Stop and remove resources that were removed from the configuration. 268210886Spjd */ 269210886Spjd TAILQ_FOREACH_SAFE(cres, &cfg->hc_resources, hr_next, tres) { 270210886Spjd TAILQ_FOREACH(nres, &newcfg->hc_resources, hr_next) { 271210886Spjd if (strcmp(cres->hr_name, nres->hr_name) == 0) 272210886Spjd break; 273210886Spjd } 274210886Spjd if (nres == NULL) { 275210886Spjd control_set_role(cres, HAST_ROLE_INIT); 276210886Spjd TAILQ_REMOVE(&cfg->hc_resources, cres, hr_next); 277210886Spjd pjdlog_info("Resource %s removed.", cres->hr_name); 278210886Spjd free(cres); 279210886Spjd } 280210886Spjd } 281210886Spjd /* 282210886Spjd * Move new resources to the current configuration. 283210886Spjd */ 284210886Spjd TAILQ_FOREACH_SAFE(nres, &newcfg->hc_resources, hr_next, tres) { 285210886Spjd TAILQ_FOREACH(cres, &cfg->hc_resources, hr_next) { 286210886Spjd if (strcmp(cres->hr_name, nres->hr_name) == 0) 287210886Spjd break; 288210886Spjd } 289210886Spjd if (cres == NULL) { 290210886Spjd TAILQ_REMOVE(&newcfg->hc_resources, nres, hr_next); 291210886Spjd TAILQ_INSERT_TAIL(&cfg->hc_resources, nres, hr_next); 292210886Spjd pjdlog_info("Resource %s added.", nres->hr_name); 293210886Spjd } 294210886Spjd } 295210886Spjd /* 296210886Spjd * Deal with modified resources. 297210886Spjd * Depending on what has changed exactly we might want to perform 298210886Spjd * different actions. 299210886Spjd * 300210886Spjd * We do full resource restart in the following situations: 301210886Spjd * Resource role is INIT or SECONDARY. 302210886Spjd * Resource role is PRIMARY and path to local component or provider 303210886Spjd * name has changed. 304210886Spjd * In case of PRIMARY, the worker process will be killed and restarted, 305210886Spjd * which also means removing /dev/hast/<name> provider and 306210886Spjd * recreating it. 307210886Spjd * 308210886Spjd * We do just reload (send SIGHUP to worker process) if we act as 309217729Spjd * PRIMARY, but only if remote address, replication mode, timeout or 310217729Spjd * execution path has changed. For those, there is no need to restart 311217729Spjd * worker process. 312210886Spjd * If PRIMARY receives SIGHUP, it will reconnect if remote address or 313210886Spjd * replication mode has changed or simply set new timeout if only 314210886Spjd * timeout has changed. 315210886Spjd */ 316210886Spjd TAILQ_FOREACH_SAFE(nres, &newcfg->hc_resources, hr_next, tres) { 317210886Spjd TAILQ_FOREACH(cres, &cfg->hc_resources, hr_next) { 318210886Spjd if (strcmp(cres->hr_name, nres->hr_name) == 0) 319210886Spjd break; 320210886Spjd } 321210886Spjd assert(cres != NULL); 322210886Spjd if (resource_needs_restart(cres, nres)) { 323210886Spjd pjdlog_info("Resource %s configuration was modified, restarting it.", 324210886Spjd cres->hr_name); 325210886Spjd role = cres->hr_role; 326210886Spjd control_set_role(cres, HAST_ROLE_INIT); 327210886Spjd TAILQ_REMOVE(&cfg->hc_resources, cres, hr_next); 328210886Spjd free(cres); 329210886Spjd TAILQ_REMOVE(&newcfg->hc_resources, nres, hr_next); 330210886Spjd TAILQ_INSERT_TAIL(&cfg->hc_resources, nres, hr_next); 331210886Spjd control_set_role(nres, role); 332210886Spjd } else if (resource_needs_reload(cres, nres)) { 333210886Spjd pjdlog_info("Resource %s configuration was modified, reloading it.", 334210886Spjd cres->hr_name); 335210886Spjd strlcpy(cres->hr_remoteaddr, nres->hr_remoteaddr, 336210886Spjd sizeof(cres->hr_remoteaddr)); 337210886Spjd cres->hr_replication = nres->hr_replication; 338210886Spjd cres->hr_timeout = nres->hr_timeout; 339217729Spjd strlcpy(cres->hr_exec, nres->hr_exec, 340217729Spjd sizeof(cres->hr_exec)); 341210886Spjd if (cres->hr_workerpid != 0) { 342210886Spjd if (kill(cres->hr_workerpid, SIGHUP) < 0) { 343210886Spjd pjdlog_errno(LOG_WARNING, 344210886Spjd "Unable to send SIGHUP to worker process %u", 345210886Spjd (unsigned int)cres->hr_workerpid); 346210886Spjd } 347210886Spjd } 348210886Spjd } 349210886Spjd } 350210886Spjd 351210886Spjd yy_config_free(newcfg); 352210886Spjd pjdlog_info("Configuration reloaded successfully."); 353210886Spjd return; 354210886Spjdfailed: 355210886Spjd if (newcfg != NULL) { 356210886Spjd if (newcfg->hc_controlconn != NULL) 357210886Spjd proto_close(newcfg->hc_controlconn); 358210886Spjd if (newcfg->hc_listenconn != NULL) 359210886Spjd proto_close(newcfg->hc_listenconn); 360210886Spjd yy_config_free(newcfg); 361210886Spjd } 362210886Spjd pjdlog_warning("Configuration not reloaded."); 363204076Spjd} 364204076Spjd 365204076Spjdstatic void 366211899Spjdterminate_workers(void) 367211899Spjd{ 368211899Spjd struct hast_resource *res; 369211899Spjd 370211899Spjd pjdlog_info("Termination signal received, exiting."); 371211899Spjd TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 372211899Spjd if (res->hr_workerpid == 0) 373211899Spjd continue; 374211899Spjd pjdlog_info("Terminating worker process (resource=%s, role=%s, pid=%u).", 375211899Spjd res->hr_name, role2str(res->hr_role), res->hr_workerpid); 376211899Spjd if (kill(res->hr_workerpid, SIGTERM) == 0) 377211899Spjd continue; 378211899Spjd pjdlog_errno(LOG_WARNING, 379211899Spjd "Unable to send signal to worker process (resource=%s, role=%s, pid=%u).", 380211899Spjd res->hr_name, role2str(res->hr_role), res->hr_workerpid); 381211899Spjd } 382211899Spjd} 383211899Spjd 384211899Spjdstatic void 385204076Spjdlisten_accept(void) 386204076Spjd{ 387204076Spjd struct hast_resource *res; 388204076Spjd struct proto_conn *conn; 389204076Spjd struct nv *nvin, *nvout, *nverr; 390204076Spjd const char *resname; 391204076Spjd const unsigned char *token; 392204076Spjd char laddr[256], raddr[256]; 393204076Spjd size_t size; 394204076Spjd pid_t pid; 395204076Spjd int status; 396204076Spjd 397204076Spjd proto_local_address(cfg->hc_listenconn, laddr, sizeof(laddr)); 398204076Spjd pjdlog_debug(1, "Accepting connection to %s.", laddr); 399204076Spjd 400204076Spjd if (proto_accept(cfg->hc_listenconn, &conn) < 0) { 401204076Spjd pjdlog_errno(LOG_ERR, "Unable to accept connection %s", laddr); 402204076Spjd return; 403204076Spjd } 404204076Spjd 405204076Spjd proto_local_address(conn, laddr, sizeof(laddr)); 406204076Spjd proto_remote_address(conn, raddr, sizeof(raddr)); 407209185Spjd pjdlog_info("Connection from %s to %s.", raddr, laddr); 408204076Spjd 409207371Spjd /* Error in setting timeout is not critical, but why should it fail? */ 410207371Spjd if (proto_timeout(conn, HAST_TIMEOUT) < 0) 411207371Spjd pjdlog_errno(LOG_WARNING, "Unable to set connection timeout"); 412207371Spjd 413204076Spjd nvin = nvout = nverr = NULL; 414204076Spjd 415204076Spjd /* 416204076Spjd * Before receiving any data see if remote host have access to any 417204076Spjd * resource. 418204076Spjd */ 419204076Spjd TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 420204076Spjd if (proto_address_match(conn, res->hr_remoteaddr)) 421204076Spjd break; 422204076Spjd } 423204076Spjd if (res == NULL) { 424204076Spjd pjdlog_error("Client %s isn't known.", raddr); 425204076Spjd goto close; 426204076Spjd } 427204076Spjd /* Ok, remote host can access at least one resource. */ 428204076Spjd 429204076Spjd if (hast_proto_recv_hdr(conn, &nvin) < 0) { 430204076Spjd pjdlog_errno(LOG_ERR, "Unable to receive header from %s", 431204076Spjd raddr); 432204076Spjd goto close; 433204076Spjd } 434204076Spjd 435204076Spjd resname = nv_get_string(nvin, "resource"); 436204076Spjd if (resname == NULL) { 437204076Spjd pjdlog_error("No 'resource' field in the header received from %s.", 438204076Spjd raddr); 439204076Spjd goto close; 440204076Spjd } 441204076Spjd pjdlog_debug(2, "%s: resource=%s", raddr, resname); 442204076Spjd token = nv_get_uint8_array(nvin, &size, "token"); 443204076Spjd /* 444204076Spjd * NULL token means that this is first conection. 445204076Spjd */ 446204076Spjd if (token != NULL && size != sizeof(res->hr_token)) { 447204076Spjd pjdlog_error("Received token of invalid size from %s (expected %zu, got %zu).", 448204076Spjd raddr, sizeof(res->hr_token), size); 449204076Spjd goto close; 450204076Spjd } 451204076Spjd 452204076Spjd /* 453204076Spjd * From now on we want to send errors to the remote node. 454204076Spjd */ 455204076Spjd nverr = nv_alloc(); 456204076Spjd 457204076Spjd /* Find resource related to this connection. */ 458204076Spjd TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 459204076Spjd if (strcmp(resname, res->hr_name) == 0) 460204076Spjd break; 461204076Spjd } 462204076Spjd /* Have we found the resource? */ 463204076Spjd if (res == NULL) { 464204076Spjd pjdlog_error("No resource '%s' as requested by %s.", 465204076Spjd resname, raddr); 466204076Spjd nv_add_stringf(nverr, "errmsg", "Resource not configured."); 467204076Spjd goto fail; 468204076Spjd } 469204076Spjd 470204076Spjd /* Now that we know resource name setup log prefix. */ 471204076Spjd pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role)); 472204076Spjd 473204076Spjd /* Does the remote host have access to this resource? */ 474204076Spjd if (!proto_address_match(conn, res->hr_remoteaddr)) { 475204076Spjd pjdlog_error("Client %s has no access to the resource.", raddr); 476204076Spjd nv_add_stringf(nverr, "errmsg", "No access to the resource."); 477204076Spjd goto fail; 478204076Spjd } 479204076Spjd /* Is the resource marked as secondary? */ 480204076Spjd if (res->hr_role != HAST_ROLE_SECONDARY) { 481204076Spjd pjdlog_error("We act as %s for the resource and not as %s as requested by %s.", 482204076Spjd role2str(res->hr_role), role2str(HAST_ROLE_SECONDARY), 483204076Spjd raddr); 484204076Spjd nv_add_stringf(nverr, "errmsg", 485204076Spjd "Remote node acts as %s for the resource and not as %s.", 486204076Spjd role2str(res->hr_role), role2str(HAST_ROLE_SECONDARY)); 487204076Spjd goto fail; 488204076Spjd } 489204076Spjd /* Does token (if exists) match? */ 490204076Spjd if (token != NULL && memcmp(token, res->hr_token, 491204076Spjd sizeof(res->hr_token)) != 0) { 492204076Spjd pjdlog_error("Token received from %s doesn't match.", raddr); 493209185Spjd nv_add_stringf(nverr, "errmsg", "Token doesn't match."); 494204076Spjd goto fail; 495204076Spjd } 496204076Spjd /* 497204076Spjd * If there is no token, but we have half-open connection 498204076Spjd * (only remotein) or full connection (worker process is running) 499204076Spjd * we have to cancel those and accept the new connection. 500204076Spjd */ 501204076Spjd if (token == NULL) { 502204076Spjd assert(res->hr_remoteout == NULL); 503204076Spjd pjdlog_debug(1, "Initial connection from %s.", raddr); 504204076Spjd if (res->hr_workerpid != 0) { 505204076Spjd assert(res->hr_remotein == NULL); 506204076Spjd pjdlog_debug(1, 507204076Spjd "Worker process exists (pid=%u), stopping it.", 508204076Spjd (unsigned int)res->hr_workerpid); 509204076Spjd /* Stop child process. */ 510204076Spjd if (kill(res->hr_workerpid, SIGINT) < 0) { 511204076Spjd pjdlog_errno(LOG_ERR, 512204076Spjd "Unable to stop worker process (pid=%u)", 513204076Spjd (unsigned int)res->hr_workerpid); 514204076Spjd /* 515204076Spjd * Other than logging the problem we 516204076Spjd * ignore it - nothing smart to do. 517204076Spjd */ 518204076Spjd } 519204076Spjd /* Wait for it to exit. */ 520204076Spjd else if ((pid = waitpid(res->hr_workerpid, 521204076Spjd &status, 0)) != res->hr_workerpid) { 522207372Spjd /* We can only log the problem. */ 523204076Spjd pjdlog_errno(LOG_ERR, 524204076Spjd "Waiting for worker process (pid=%u) failed", 525204076Spjd (unsigned int)res->hr_workerpid); 526204076Spjd } else { 527207372Spjd child_exit_log(res->hr_workerpid, status); 528204076Spjd } 529213006Spjd child_cleanup(res); 530204076Spjd } else if (res->hr_remotein != NULL) { 531204076Spjd char oaddr[256]; 532204076Spjd 533213981Spjd proto_remote_address(res->hr_remotein, oaddr, 534213981Spjd sizeof(oaddr)); 535204076Spjd pjdlog_debug(1, 536204076Spjd "Canceling half-open connection from %s on connection from %s.", 537204076Spjd oaddr, raddr); 538204076Spjd proto_close(res->hr_remotein); 539204076Spjd res->hr_remotein = NULL; 540204076Spjd } 541204076Spjd } 542204076Spjd 543204076Spjd /* 544204076Spjd * Checks and cleanups are done. 545204076Spjd */ 546204076Spjd 547204076Spjd if (token == NULL) { 548204076Spjd arc4random_buf(res->hr_token, sizeof(res->hr_token)); 549204076Spjd nvout = nv_alloc(); 550204076Spjd nv_add_uint8_array(nvout, res->hr_token, 551204076Spjd sizeof(res->hr_token), "token"); 552204076Spjd if (nv_error(nvout) != 0) { 553204076Spjd pjdlog_common(LOG_ERR, 0, nv_error(nvout), 554204076Spjd "Unable to prepare return header for %s", raddr); 555204076Spjd nv_add_stringf(nverr, "errmsg", 556204076Spjd "Remote node was unable to prepare return header: %s.", 557204076Spjd strerror(nv_error(nvout))); 558204076Spjd goto fail; 559204076Spjd } 560204076Spjd if (hast_proto_send(NULL, conn, nvout, NULL, 0) < 0) { 561204076Spjd int error = errno; 562204076Spjd 563204076Spjd pjdlog_errno(LOG_ERR, "Unable to send response to %s", 564204076Spjd raddr); 565204076Spjd nv_add_stringf(nverr, "errmsg", 566204076Spjd "Remote node was unable to send response: %s.", 567204076Spjd strerror(error)); 568204076Spjd goto fail; 569204076Spjd } 570204076Spjd res->hr_remotein = conn; 571204076Spjd pjdlog_debug(1, "Incoming connection from %s configured.", 572204076Spjd raddr); 573204076Spjd } else { 574204076Spjd res->hr_remoteout = conn; 575204076Spjd pjdlog_debug(1, "Outgoing connection to %s configured.", raddr); 576204076Spjd hastd_secondary(res, nvin); 577204076Spjd } 578204076Spjd nv_free(nvin); 579204076Spjd nv_free(nvout); 580204076Spjd nv_free(nverr); 581204076Spjd pjdlog_prefix_set("%s", ""); 582204076Spjd return; 583204076Spjdfail: 584204076Spjd if (nv_error(nverr) != 0) { 585204076Spjd pjdlog_common(LOG_ERR, 0, nv_error(nverr), 586204076Spjd "Unable to prepare error header for %s", raddr); 587204076Spjd goto close; 588204076Spjd } 589204076Spjd if (hast_proto_send(NULL, conn, nverr, NULL, 0) < 0) { 590204076Spjd pjdlog_errno(LOG_ERR, "Unable to send error to %s", raddr); 591204076Spjd goto close; 592204076Spjd } 593204076Spjdclose: 594204076Spjd if (nvin != NULL) 595204076Spjd nv_free(nvin); 596204076Spjd if (nvout != NULL) 597204076Spjd nv_free(nvout); 598204076Spjd if (nverr != NULL) 599204076Spjd nv_free(nverr); 600204076Spjd proto_close(conn); 601204076Spjd pjdlog_prefix_set("%s", ""); 602204076Spjd} 603204076Spjd 604204076Spjdstatic void 605204076Spjdmain_loop(void) 606204076Spjd{ 607212038Spjd struct hast_resource *res; 608213009Spjd struct timeval seltimeout; 609213009Spjd struct timespec sigtimeout; 610213009Spjd int fd, maxfd, ret, signo; 611213009Spjd sigset_t mask; 612212037Spjd fd_set rfds; 613204076Spjd 614213009Spjd seltimeout.tv_sec = REPORT_INTERVAL; 615213009Spjd seltimeout.tv_usec = 0; 616213009Spjd sigtimeout.tv_sec = 0; 617213009Spjd sigtimeout.tv_nsec = 0; 618211977Spjd 619213009Spjd PJDLOG_VERIFY(sigemptyset(&mask) == 0); 620213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGHUP) == 0); 621213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGINT) == 0); 622213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGTERM) == 0); 623213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGCHLD) == 0); 624213009Spjd 625216477Spjd pjdlog_info("Started successfully, running protocol version %d.", 626216477Spjd HAST_PROTO_VERSION); 627216477Spjd 628204076Spjd for (;;) { 629213009Spjd while ((signo = sigtimedwait(&mask, NULL, &sigtimeout)) != -1) { 630213009Spjd switch (signo) { 631213009Spjd case SIGINT: 632213009Spjd case SIGTERM: 633213009Spjd sigexit_received = true; 634213009Spjd terminate_workers(); 635213009Spjd exit(EX_OK); 636213009Spjd break; 637213009Spjd case SIGCHLD: 638213009Spjd child_exit(); 639213009Spjd break; 640213009Spjd case SIGHUP: 641213009Spjd hastd_reload(); 642213009Spjd break; 643213009Spjd default: 644213009Spjd assert(!"invalid condition"); 645213009Spjd } 646211899Spjd } 647204076Spjd 648209177Spjd /* Setup descriptors for select(2). */ 649204076Spjd FD_ZERO(&rfds); 650212038Spjd maxfd = fd = proto_descriptor(cfg->hc_controlconn); 651213008Spjd assert(fd >= 0); 652212038Spjd FD_SET(fd, &rfds); 653212038Spjd fd = proto_descriptor(cfg->hc_listenconn); 654213008Spjd assert(fd >= 0); 655212038Spjd FD_SET(fd, &rfds); 656212038Spjd maxfd = fd > maxfd ? fd : maxfd; 657212038Spjd TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 658212038Spjd if (res->hr_event == NULL) 659212038Spjd continue; 660212038Spjd fd = proto_descriptor(res->hr_event); 661213008Spjd assert(fd >= 0); 662212038Spjd FD_SET(fd, &rfds); 663212038Spjd maxfd = fd > maxfd ? fd : maxfd; 664212038Spjd } 665204076Spjd 666213008Spjd assert(maxfd + 1 <= (int)FD_SETSIZE); 667213009Spjd ret = select(maxfd + 1, &rfds, NULL, NULL, &seltimeout); 668211977Spjd if (ret == 0) 669213429Spjd hook_check(); 670211977Spjd else if (ret == -1) { 671204076Spjd if (errno == EINTR) 672204076Spjd continue; 673204076Spjd KEEP_ERRNO((void)pidfile_remove(pfh)); 674204076Spjd pjdlog_exit(EX_OSERR, "select() failed"); 675204076Spjd } 676204076Spjd 677212038Spjd if (FD_ISSET(proto_descriptor(cfg->hc_controlconn), &rfds)) 678204076Spjd control_handle(cfg); 679212038Spjd if (FD_ISSET(proto_descriptor(cfg->hc_listenconn), &rfds)) 680204076Spjd listen_accept(); 681212038Spjd TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 682212038Spjd if (res->hr_event == NULL) 683212038Spjd continue; 684212038Spjd if (FD_ISSET(proto_descriptor(res->hr_event), &rfds)) { 685212038Spjd if (event_recv(res) == 0) 686212038Spjd continue; 687212038Spjd /* The worker process exited? */ 688212038Spjd proto_close(res->hr_event); 689212038Spjd res->hr_event = NULL; 690212038Spjd } 691212038Spjd } 692204076Spjd } 693204076Spjd} 694204076Spjd 695213428Spjdstatic void 696213428Spjddummy_sighandler(int sig __unused) 697213428Spjd{ 698213428Spjd /* Nothing to do. */ 699213428Spjd} 700213428Spjd 701204076Spjdint 702204076Spjdmain(int argc, char *argv[]) 703204076Spjd{ 704204076Spjd const char *pidfile; 705204076Spjd pid_t otherpid; 706204076Spjd bool foreground; 707204076Spjd int debuglevel; 708213009Spjd sigset_t mask; 709204076Spjd 710204076Spjd foreground = false; 711204076Spjd debuglevel = 0; 712204076Spjd pidfile = HASTD_PIDFILE; 713204076Spjd 714204076Spjd for (;;) { 715204076Spjd int ch; 716204076Spjd 717204076Spjd ch = getopt(argc, argv, "c:dFhP:"); 718204076Spjd if (ch == -1) 719204076Spjd break; 720204076Spjd switch (ch) { 721204076Spjd case 'c': 722204076Spjd cfgpath = optarg; 723204076Spjd break; 724204076Spjd case 'd': 725204076Spjd debuglevel++; 726204076Spjd break; 727204076Spjd case 'F': 728204076Spjd foreground = true; 729204076Spjd break; 730204076Spjd case 'P': 731204076Spjd pidfile = optarg; 732204076Spjd break; 733204076Spjd case 'h': 734204076Spjd default: 735204076Spjd usage(); 736204076Spjd } 737204076Spjd } 738204076Spjd argc -= optind; 739204076Spjd argv += optind; 740204076Spjd 741204076Spjd pjdlog_debug_set(debuglevel); 742204076Spjd 743214273Spjd g_gate_load(); 744214273Spjd 745204076Spjd pfh = pidfile_open(pidfile, 0600, &otherpid); 746204076Spjd if (pfh == NULL) { 747204076Spjd if (errno == EEXIST) { 748204076Spjd pjdlog_exitx(EX_TEMPFAIL, 749204076Spjd "Another hastd is already running, pid: %jd.", 750204076Spjd (intmax_t)otherpid); 751204076Spjd } 752204076Spjd /* If we cannot create pidfile from other reasons, only warn. */ 753210879Spjd pjdlog_errno(LOG_WARNING, "Unable to open or create pidfile"); 754204076Spjd } 755204076Spjd 756210883Spjd cfg = yy_config_parse(cfgpath, true); 757204076Spjd assert(cfg != NULL); 758204076Spjd 759213428Spjd /* 760217307Spjd * Restore default actions for interesting signals in case parent 761217307Spjd * process (like init(8)) decided to ignore some of them (like SIGHUP). 762217307Spjd */ 763217307Spjd PJDLOG_VERIFY(signal(SIGHUP, SIG_DFL) != SIG_ERR); 764217307Spjd PJDLOG_VERIFY(signal(SIGINT, SIG_DFL) != SIG_ERR); 765217307Spjd PJDLOG_VERIFY(signal(SIGTERM, SIG_DFL) != SIG_ERR); 766217307Spjd /* 767213428Spjd * Because SIGCHLD is ignored by default, setup dummy handler for it, 768213428Spjd * so we can mask it. 769213428Spjd */ 770213428Spjd PJDLOG_VERIFY(signal(SIGCHLD, dummy_sighandler) != SIG_ERR); 771217307Spjd 772213009Spjd PJDLOG_VERIFY(sigemptyset(&mask) == 0); 773213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGHUP) == 0); 774213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGINT) == 0); 775213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGTERM) == 0); 776213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGCHLD) == 0); 777213009Spjd PJDLOG_VERIFY(sigprocmask(SIG_SETMASK, &mask, NULL) == 0); 778204076Spjd 779204076Spjd /* Listen on control address. */ 780204076Spjd if (proto_server(cfg->hc_controladdr, &cfg->hc_controlconn) < 0) { 781204076Spjd KEEP_ERRNO((void)pidfile_remove(pfh)); 782204076Spjd pjdlog_exit(EX_OSERR, "Unable to listen on control address %s", 783204076Spjd cfg->hc_controladdr); 784204076Spjd } 785204076Spjd /* Listen for remote connections. */ 786204076Spjd if (proto_server(cfg->hc_listenaddr, &cfg->hc_listenconn) < 0) { 787204076Spjd KEEP_ERRNO((void)pidfile_remove(pfh)); 788204076Spjd pjdlog_exit(EX_OSERR, "Unable to listen on address %s", 789204076Spjd cfg->hc_listenaddr); 790204076Spjd } 791204076Spjd 792204076Spjd if (!foreground) { 793204076Spjd if (daemon(0, 0) < 0) { 794204076Spjd KEEP_ERRNO((void)pidfile_remove(pfh)); 795204076Spjd pjdlog_exit(EX_OSERR, "Unable to daemonize"); 796204076Spjd } 797204076Spjd 798204076Spjd /* Start logging to syslog. */ 799204076Spjd pjdlog_mode_set(PJDLOG_MODE_SYSLOG); 800204076Spjd 801204076Spjd /* Write PID to a file. */ 802204076Spjd if (pidfile_write(pfh) < 0) { 803204076Spjd pjdlog_errno(LOG_WARNING, 804204076Spjd "Unable to write PID to a file"); 805204076Spjd } 806204076Spjd } 807204076Spjd 808211977Spjd hook_init(); 809211977Spjd 810204076Spjd main_loop(); 811204076Spjd 812204076Spjd exit(0); 813204076Spjd} 814