hastd.c revision 213430
1204076Spjd/*- 2204076Spjd * Copyright (c) 2009-2010 The FreeBSD Foundation 3210886Spjd * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org> 4204076Spjd * All rights reserved. 5204076Spjd * 6204076Spjd * This software was developed by Pawel Jakub Dawidek under sponsorship from 7204076Spjd * the FreeBSD Foundation. 8204076Spjd * 9204076Spjd * Redistribution and use in source and binary forms, with or without 10204076Spjd * modification, are permitted provided that the following conditions 11204076Spjd * are met: 12204076Spjd * 1. Redistributions of source code must retain the above copyright 13204076Spjd * notice, this list of conditions and the following disclaimer. 14204076Spjd * 2. Redistributions in binary form must reproduce the above copyright 15204076Spjd * notice, this list of conditions and the following disclaimer in the 16204076Spjd * documentation and/or other materials provided with the distribution. 17204076Spjd * 18204076Spjd * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 19204076Spjd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20204076Spjd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21204076Spjd * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 22204076Spjd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23204076Spjd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24204076Spjd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25204076Spjd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26204076Spjd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27204076Spjd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28204076Spjd * SUCH DAMAGE. 29204076Spjd */ 30204076Spjd 31204076Spjd#include <sys/cdefs.h> 32204076Spjd__FBSDID("$FreeBSD: head/sbin/hastd/hastd.c 213430 2010-10-04 21:44:26Z pjd $"); 33204076Spjd 34204076Spjd#include <sys/param.h> 35204076Spjd#include <sys/linker.h> 36204076Spjd#include <sys/module.h> 37204076Spjd#include <sys/wait.h> 38204076Spjd 39204076Spjd#include <assert.h> 40204076Spjd#include <err.h> 41204076Spjd#include <errno.h> 42204076Spjd#include <libutil.h> 43204076Spjd#include <signal.h> 44204076Spjd#include <stdbool.h> 45204076Spjd#include <stdio.h> 46204076Spjd#include <stdlib.h> 47204076Spjd#include <string.h> 48204076Spjd#include <sysexits.h> 49204076Spjd#include <unistd.h> 50204076Spjd 51204076Spjd#include <activemap.h> 52204076Spjd#include <pjdlog.h> 53204076Spjd 54204076Spjd#include "control.h" 55212038Spjd#include "event.h" 56204076Spjd#include "hast.h" 57204076Spjd#include "hast_proto.h" 58204076Spjd#include "hastd.h" 59211977Spjd#include "hooks.h" 60204076Spjd#include "subr.h" 61204076Spjd 62204076Spjd/* Path to configuration file. */ 63210886Spjdconst char *cfgpath = HAST_CONFIG; 64204076Spjd/* Hastd configuration. */ 65204076Spjdstatic struct hastd_config *cfg; 66204076Spjd/* Was SIGINT or SIGTERM signal received? */ 67204076Spjdbool sigexit_received = false; 68204076Spjd/* PID file handle. */ 69204076Spjdstruct pidfh *pfh; 70204076Spjd 71211977Spjd/* How often check for hooks running for too long. */ 72213430Spjd#define REPORT_INTERVAL 5 73211977Spjd 74204076Spjdstatic void 75204076Spjdusage(void) 76204076Spjd{ 77204076Spjd 78204076Spjd errx(EX_USAGE, "[-dFh] [-c config] [-P pidfile]"); 79204076Spjd} 80204076Spjd 81204076Spjdstatic void 82204076Spjdg_gate_load(void) 83204076Spjd{ 84204076Spjd 85204076Spjd if (modfind("g_gate") == -1) { 86204076Spjd /* Not present in kernel, try loading it. */ 87204076Spjd if (kldload("geom_gate") == -1 || modfind("g_gate") == -1) { 88204076Spjd if (errno != EEXIST) { 89204076Spjd pjdlog_exit(EX_OSERR, 90204076Spjd "Unable to load geom_gate module"); 91204076Spjd } 92204076Spjd } 93204076Spjd } 94204076Spjd} 95204076Spjd 96204076Spjdstatic void 97207372Spjdchild_exit_log(unsigned int pid, int status) 98207372Spjd{ 99207372Spjd 100207372Spjd if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { 101207372Spjd pjdlog_debug(1, "Worker process exited gracefully (pid=%u).", 102207372Spjd pid); 103207372Spjd } else if (WIFSIGNALED(status)) { 104207372Spjd pjdlog_error("Worker process killed (pid=%u, signal=%d).", 105207372Spjd pid, WTERMSIG(status)); 106207372Spjd } else { 107207372Spjd pjdlog_error("Worker process exited ungracefully (pid=%u, exitcode=%d).", 108207372Spjd pid, WIFEXITED(status) ? WEXITSTATUS(status) : -1); 109207372Spjd } 110207372Spjd} 111207372Spjd 112207372Spjdstatic void 113204076Spjdchild_exit(void) 114204076Spjd{ 115204076Spjd struct hast_resource *res; 116204076Spjd int status; 117204076Spjd pid_t pid; 118204076Spjd 119204076Spjd while ((pid = wait3(&status, WNOHANG, NULL)) > 0) { 120204076Spjd /* Find resource related to the process that just exited. */ 121204076Spjd TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 122204076Spjd if (pid == res->hr_workerpid) 123204076Spjd break; 124204076Spjd } 125204076Spjd if (res == NULL) { 126204076Spjd /* 127204076Spjd * This can happen when new connection arrives and we 128211977Spjd * cancel child responsible for the old one or if this 129211977Spjd * was hook which we executed. 130204076Spjd */ 131211977Spjd hook_check_one(pid, status); 132204076Spjd continue; 133204076Spjd } 134204076Spjd pjdlog_prefix_set("[%s] (%s) ", res->hr_name, 135204076Spjd role2str(res->hr_role)); 136207372Spjd child_exit_log(pid, status); 137213006Spjd child_cleanup(res); 138204076Spjd if (res->hr_role == HAST_ROLE_PRIMARY) { 139207372Spjd /* 140207372Spjd * Restart child process if it was killed by signal 141207372Spjd * or exited because of temporary problem. 142207372Spjd */ 143207372Spjd if (WIFSIGNALED(status) || 144207372Spjd (WIFEXITED(status) && 145207372Spjd WEXITSTATUS(status) == EX_TEMPFAIL)) { 146207348Spjd sleep(1); 147207348Spjd pjdlog_info("Restarting worker process."); 148207348Spjd hastd_primary(res); 149207348Spjd } else { 150207348Spjd res->hr_role = HAST_ROLE_INIT; 151207348Spjd pjdlog_info("Changing resource role back to %s.", 152207348Spjd role2str(res->hr_role)); 153207348Spjd } 154204076Spjd } 155204076Spjd pjdlog_prefix_set("%s", ""); 156204076Spjd } 157204076Spjd} 158204076Spjd 159210886Spjdstatic bool 160210886Spjdresource_needs_restart(const struct hast_resource *res0, 161210886Spjd const struct hast_resource *res1) 162210886Spjd{ 163210886Spjd 164210886Spjd assert(strcmp(res0->hr_name, res1->hr_name) == 0); 165210886Spjd 166210886Spjd if (strcmp(res0->hr_provname, res1->hr_provname) != 0) 167210886Spjd return (true); 168210886Spjd if (strcmp(res0->hr_localpath, res1->hr_localpath) != 0) 169210886Spjd return (true); 170210886Spjd if (res0->hr_role == HAST_ROLE_INIT || 171210886Spjd res0->hr_role == HAST_ROLE_SECONDARY) { 172210886Spjd if (strcmp(res0->hr_remoteaddr, res1->hr_remoteaddr) != 0) 173210886Spjd return (true); 174210886Spjd if (res0->hr_replication != res1->hr_replication) 175210886Spjd return (true); 176210886Spjd if (res0->hr_timeout != res1->hr_timeout) 177210886Spjd return (true); 178211886Spjd if (strcmp(res0->hr_exec, res1->hr_exec) != 0) 179211886Spjd return (true); 180210886Spjd } 181210886Spjd return (false); 182210886Spjd} 183210886Spjd 184210886Spjdstatic bool 185210886Spjdresource_needs_reload(const struct hast_resource *res0, 186210886Spjd const struct hast_resource *res1) 187210886Spjd{ 188210886Spjd 189210886Spjd assert(strcmp(res0->hr_name, res1->hr_name) == 0); 190210886Spjd assert(strcmp(res0->hr_provname, res1->hr_provname) == 0); 191210886Spjd assert(strcmp(res0->hr_localpath, res1->hr_localpath) == 0); 192210886Spjd 193210886Spjd if (res0->hr_role != HAST_ROLE_PRIMARY) 194210886Spjd return (false); 195210886Spjd 196210886Spjd if (strcmp(res0->hr_remoteaddr, res1->hr_remoteaddr) != 0) 197210886Spjd return (true); 198210886Spjd if (res0->hr_replication != res1->hr_replication) 199210886Spjd return (true); 200210886Spjd if (res0->hr_timeout != res1->hr_timeout) 201210886Spjd return (true); 202211886Spjd if (strcmp(res0->hr_exec, res1->hr_exec) != 0) 203211886Spjd return (true); 204210886Spjd return (false); 205210886Spjd} 206210886Spjd 207204076Spjdstatic void 208204076Spjdhastd_reload(void) 209204076Spjd{ 210210886Spjd struct hastd_config *newcfg; 211210886Spjd struct hast_resource *nres, *cres, *tres; 212210886Spjd uint8_t role; 213204076Spjd 214210886Spjd pjdlog_info("Reloading configuration..."); 215210886Spjd 216210886Spjd newcfg = yy_config_parse(cfgpath, false); 217210886Spjd if (newcfg == NULL) 218210886Spjd goto failed; 219210886Spjd 220210886Spjd /* 221210886Spjd * Check if control address has changed. 222210886Spjd */ 223210886Spjd if (strcmp(cfg->hc_controladdr, newcfg->hc_controladdr) != 0) { 224210886Spjd if (proto_server(newcfg->hc_controladdr, 225210886Spjd &newcfg->hc_controlconn) < 0) { 226210886Spjd pjdlog_errno(LOG_ERR, 227210886Spjd "Unable to listen on control address %s", 228210886Spjd newcfg->hc_controladdr); 229210886Spjd goto failed; 230210886Spjd } 231210886Spjd } 232210886Spjd /* 233210886Spjd * Check if listen address has changed. 234210886Spjd */ 235210886Spjd if (strcmp(cfg->hc_listenaddr, newcfg->hc_listenaddr) != 0) { 236210886Spjd if (proto_server(newcfg->hc_listenaddr, 237210886Spjd &newcfg->hc_listenconn) < 0) { 238210886Spjd pjdlog_errno(LOG_ERR, "Unable to listen on address %s", 239210886Spjd newcfg->hc_listenaddr); 240210886Spjd goto failed; 241210886Spjd } 242210886Spjd } 243210886Spjd /* 244210886Spjd * Only when both control and listen sockets are successfully 245210886Spjd * initialized switch them to new configuration. 246210886Spjd */ 247210886Spjd if (newcfg->hc_controlconn != NULL) { 248210886Spjd pjdlog_info("Control socket changed from %s to %s.", 249210886Spjd cfg->hc_controladdr, newcfg->hc_controladdr); 250210886Spjd proto_close(cfg->hc_controlconn); 251210886Spjd cfg->hc_controlconn = newcfg->hc_controlconn; 252210886Spjd newcfg->hc_controlconn = NULL; 253210886Spjd strlcpy(cfg->hc_controladdr, newcfg->hc_controladdr, 254210886Spjd sizeof(cfg->hc_controladdr)); 255210886Spjd } 256210886Spjd if (newcfg->hc_listenconn != NULL) { 257210886Spjd pjdlog_info("Listen socket changed from %s to %s.", 258210886Spjd cfg->hc_listenaddr, newcfg->hc_listenaddr); 259210886Spjd proto_close(cfg->hc_listenconn); 260210886Spjd cfg->hc_listenconn = newcfg->hc_listenconn; 261210886Spjd newcfg->hc_listenconn = NULL; 262210886Spjd strlcpy(cfg->hc_listenaddr, newcfg->hc_listenaddr, 263210886Spjd sizeof(cfg->hc_listenaddr)); 264210886Spjd } 265210886Spjd 266210886Spjd /* 267210886Spjd * Stop and remove resources that were removed from the configuration. 268210886Spjd */ 269210886Spjd TAILQ_FOREACH_SAFE(cres, &cfg->hc_resources, hr_next, tres) { 270210886Spjd TAILQ_FOREACH(nres, &newcfg->hc_resources, hr_next) { 271210886Spjd if (strcmp(cres->hr_name, nres->hr_name) == 0) 272210886Spjd break; 273210886Spjd } 274210886Spjd if (nres == NULL) { 275210886Spjd control_set_role(cres, HAST_ROLE_INIT); 276210886Spjd TAILQ_REMOVE(&cfg->hc_resources, cres, hr_next); 277210886Spjd pjdlog_info("Resource %s removed.", cres->hr_name); 278210886Spjd free(cres); 279210886Spjd } 280210886Spjd } 281210886Spjd /* 282210886Spjd * Move new resources to the current configuration. 283210886Spjd */ 284210886Spjd TAILQ_FOREACH_SAFE(nres, &newcfg->hc_resources, hr_next, tres) { 285210886Spjd TAILQ_FOREACH(cres, &cfg->hc_resources, hr_next) { 286210886Spjd if (strcmp(cres->hr_name, nres->hr_name) == 0) 287210886Spjd break; 288210886Spjd } 289210886Spjd if (cres == NULL) { 290210886Spjd TAILQ_REMOVE(&newcfg->hc_resources, nres, hr_next); 291210886Spjd TAILQ_INSERT_TAIL(&cfg->hc_resources, nres, hr_next); 292210886Spjd pjdlog_info("Resource %s added.", nres->hr_name); 293210886Spjd } 294210886Spjd } 295210886Spjd /* 296210886Spjd * Deal with modified resources. 297210886Spjd * Depending on what has changed exactly we might want to perform 298210886Spjd * different actions. 299210886Spjd * 300210886Spjd * We do full resource restart in the following situations: 301210886Spjd * Resource role is INIT or SECONDARY. 302210886Spjd * Resource role is PRIMARY and path to local component or provider 303210886Spjd * name has changed. 304210886Spjd * In case of PRIMARY, the worker process will be killed and restarted, 305210886Spjd * which also means removing /dev/hast/<name> provider and 306210886Spjd * recreating it. 307210886Spjd * 308210886Spjd * We do just reload (send SIGHUP to worker process) if we act as 309210886Spjd * PRIMARY, but only remote address, replication mode and timeout 310210886Spjd * has changed. For those, there is no need to restart worker process. 311210886Spjd * If PRIMARY receives SIGHUP, it will reconnect if remote address or 312210886Spjd * replication mode has changed or simply set new timeout if only 313210886Spjd * timeout has changed. 314210886Spjd */ 315210886Spjd TAILQ_FOREACH_SAFE(nres, &newcfg->hc_resources, hr_next, tres) { 316210886Spjd TAILQ_FOREACH(cres, &cfg->hc_resources, hr_next) { 317210886Spjd if (strcmp(cres->hr_name, nres->hr_name) == 0) 318210886Spjd break; 319210886Spjd } 320210886Spjd assert(cres != NULL); 321210886Spjd if (resource_needs_restart(cres, nres)) { 322210886Spjd pjdlog_info("Resource %s configuration was modified, restarting it.", 323210886Spjd cres->hr_name); 324210886Spjd role = cres->hr_role; 325210886Spjd control_set_role(cres, HAST_ROLE_INIT); 326210886Spjd TAILQ_REMOVE(&cfg->hc_resources, cres, hr_next); 327210886Spjd free(cres); 328210886Spjd TAILQ_REMOVE(&newcfg->hc_resources, nres, hr_next); 329210886Spjd TAILQ_INSERT_TAIL(&cfg->hc_resources, nres, hr_next); 330210886Spjd control_set_role(nres, role); 331210886Spjd } else if (resource_needs_reload(cres, nres)) { 332210886Spjd pjdlog_info("Resource %s configuration was modified, reloading it.", 333210886Spjd cres->hr_name); 334210886Spjd strlcpy(cres->hr_remoteaddr, nres->hr_remoteaddr, 335210886Spjd sizeof(cres->hr_remoteaddr)); 336210886Spjd cres->hr_replication = nres->hr_replication; 337210886Spjd cres->hr_timeout = nres->hr_timeout; 338210886Spjd if (cres->hr_workerpid != 0) { 339210886Spjd if (kill(cres->hr_workerpid, SIGHUP) < 0) { 340210886Spjd pjdlog_errno(LOG_WARNING, 341210886Spjd "Unable to send SIGHUP to worker process %u", 342210886Spjd (unsigned int)cres->hr_workerpid); 343210886Spjd } 344210886Spjd } 345210886Spjd } 346210886Spjd } 347210886Spjd 348210886Spjd yy_config_free(newcfg); 349210886Spjd pjdlog_info("Configuration reloaded successfully."); 350210886Spjd return; 351210886Spjdfailed: 352210886Spjd if (newcfg != NULL) { 353210886Spjd if (newcfg->hc_controlconn != NULL) 354210886Spjd proto_close(newcfg->hc_controlconn); 355210886Spjd if (newcfg->hc_listenconn != NULL) 356210886Spjd proto_close(newcfg->hc_listenconn); 357210886Spjd yy_config_free(newcfg); 358210886Spjd } 359210886Spjd pjdlog_warning("Configuration not reloaded."); 360204076Spjd} 361204076Spjd 362204076Spjdstatic void 363211899Spjdterminate_workers(void) 364211899Spjd{ 365211899Spjd struct hast_resource *res; 366211899Spjd 367211899Spjd pjdlog_info("Termination signal received, exiting."); 368211899Spjd TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 369211899Spjd if (res->hr_workerpid == 0) 370211899Spjd continue; 371211899Spjd pjdlog_info("Terminating worker process (resource=%s, role=%s, pid=%u).", 372211899Spjd res->hr_name, role2str(res->hr_role), res->hr_workerpid); 373211899Spjd if (kill(res->hr_workerpid, SIGTERM) == 0) 374211899Spjd continue; 375211899Spjd pjdlog_errno(LOG_WARNING, 376211899Spjd "Unable to send signal to worker process (resource=%s, role=%s, pid=%u).", 377211899Spjd res->hr_name, role2str(res->hr_role), res->hr_workerpid); 378211899Spjd } 379211899Spjd} 380211899Spjd 381211899Spjdstatic void 382204076Spjdlisten_accept(void) 383204076Spjd{ 384204076Spjd struct hast_resource *res; 385204076Spjd struct proto_conn *conn; 386204076Spjd struct nv *nvin, *nvout, *nverr; 387204076Spjd const char *resname; 388204076Spjd const unsigned char *token; 389204076Spjd char laddr[256], raddr[256]; 390204076Spjd size_t size; 391204076Spjd pid_t pid; 392204076Spjd int status; 393204076Spjd 394204076Spjd proto_local_address(cfg->hc_listenconn, laddr, sizeof(laddr)); 395204076Spjd pjdlog_debug(1, "Accepting connection to %s.", laddr); 396204076Spjd 397204076Spjd if (proto_accept(cfg->hc_listenconn, &conn) < 0) { 398204076Spjd pjdlog_errno(LOG_ERR, "Unable to accept connection %s", laddr); 399204076Spjd return; 400204076Spjd } 401204076Spjd 402204076Spjd proto_local_address(conn, laddr, sizeof(laddr)); 403204076Spjd proto_remote_address(conn, raddr, sizeof(raddr)); 404209185Spjd pjdlog_info("Connection from %s to %s.", raddr, laddr); 405204076Spjd 406207371Spjd /* Error in setting timeout is not critical, but why should it fail? */ 407207371Spjd if (proto_timeout(conn, HAST_TIMEOUT) < 0) 408207371Spjd pjdlog_errno(LOG_WARNING, "Unable to set connection timeout"); 409207371Spjd 410204076Spjd nvin = nvout = nverr = NULL; 411204076Spjd 412204076Spjd /* 413204076Spjd * Before receiving any data see if remote host have access to any 414204076Spjd * resource. 415204076Spjd */ 416204076Spjd TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 417204076Spjd if (proto_address_match(conn, res->hr_remoteaddr)) 418204076Spjd break; 419204076Spjd } 420204076Spjd if (res == NULL) { 421204076Spjd pjdlog_error("Client %s isn't known.", raddr); 422204076Spjd goto close; 423204076Spjd } 424204076Spjd /* Ok, remote host can access at least one resource. */ 425204076Spjd 426204076Spjd if (hast_proto_recv_hdr(conn, &nvin) < 0) { 427204076Spjd pjdlog_errno(LOG_ERR, "Unable to receive header from %s", 428204076Spjd raddr); 429204076Spjd goto close; 430204076Spjd } 431204076Spjd 432204076Spjd resname = nv_get_string(nvin, "resource"); 433204076Spjd if (resname == NULL) { 434204076Spjd pjdlog_error("No 'resource' field in the header received from %s.", 435204076Spjd raddr); 436204076Spjd goto close; 437204076Spjd } 438204076Spjd pjdlog_debug(2, "%s: resource=%s", raddr, resname); 439204076Spjd token = nv_get_uint8_array(nvin, &size, "token"); 440204076Spjd /* 441204076Spjd * NULL token means that this is first conection. 442204076Spjd */ 443204076Spjd if (token != NULL && size != sizeof(res->hr_token)) { 444204076Spjd pjdlog_error("Received token of invalid size from %s (expected %zu, got %zu).", 445204076Spjd raddr, sizeof(res->hr_token), size); 446204076Spjd goto close; 447204076Spjd } 448204076Spjd 449204076Spjd /* 450204076Spjd * From now on we want to send errors to the remote node. 451204076Spjd */ 452204076Spjd nverr = nv_alloc(); 453204076Spjd 454204076Spjd /* Find resource related to this connection. */ 455204076Spjd TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 456204076Spjd if (strcmp(resname, res->hr_name) == 0) 457204076Spjd break; 458204076Spjd } 459204076Spjd /* Have we found the resource? */ 460204076Spjd if (res == NULL) { 461204076Spjd pjdlog_error("No resource '%s' as requested by %s.", 462204076Spjd resname, raddr); 463204076Spjd nv_add_stringf(nverr, "errmsg", "Resource not configured."); 464204076Spjd goto fail; 465204076Spjd } 466204076Spjd 467204076Spjd /* Now that we know resource name setup log prefix. */ 468204076Spjd pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role)); 469204076Spjd 470204076Spjd /* Does the remote host have access to this resource? */ 471204076Spjd if (!proto_address_match(conn, res->hr_remoteaddr)) { 472204076Spjd pjdlog_error("Client %s has no access to the resource.", raddr); 473204076Spjd nv_add_stringf(nverr, "errmsg", "No access to the resource."); 474204076Spjd goto fail; 475204076Spjd } 476204076Spjd /* Is the resource marked as secondary? */ 477204076Spjd if (res->hr_role != HAST_ROLE_SECONDARY) { 478204076Spjd pjdlog_error("We act as %s for the resource and not as %s as requested by %s.", 479204076Spjd role2str(res->hr_role), role2str(HAST_ROLE_SECONDARY), 480204076Spjd raddr); 481204076Spjd nv_add_stringf(nverr, "errmsg", 482204076Spjd "Remote node acts as %s for the resource and not as %s.", 483204076Spjd role2str(res->hr_role), role2str(HAST_ROLE_SECONDARY)); 484204076Spjd goto fail; 485204076Spjd } 486204076Spjd /* Does token (if exists) match? */ 487204076Spjd if (token != NULL && memcmp(token, res->hr_token, 488204076Spjd sizeof(res->hr_token)) != 0) { 489204076Spjd pjdlog_error("Token received from %s doesn't match.", raddr); 490209185Spjd nv_add_stringf(nverr, "errmsg", "Token doesn't match."); 491204076Spjd goto fail; 492204076Spjd } 493204076Spjd /* 494204076Spjd * If there is no token, but we have half-open connection 495204076Spjd * (only remotein) or full connection (worker process is running) 496204076Spjd * we have to cancel those and accept the new connection. 497204076Spjd */ 498204076Spjd if (token == NULL) { 499204076Spjd assert(res->hr_remoteout == NULL); 500204076Spjd pjdlog_debug(1, "Initial connection from %s.", raddr); 501204076Spjd if (res->hr_workerpid != 0) { 502204076Spjd assert(res->hr_remotein == NULL); 503204076Spjd pjdlog_debug(1, 504204076Spjd "Worker process exists (pid=%u), stopping it.", 505204076Spjd (unsigned int)res->hr_workerpid); 506204076Spjd /* Stop child process. */ 507204076Spjd if (kill(res->hr_workerpid, SIGINT) < 0) { 508204076Spjd pjdlog_errno(LOG_ERR, 509204076Spjd "Unable to stop worker process (pid=%u)", 510204076Spjd (unsigned int)res->hr_workerpid); 511204076Spjd /* 512204076Spjd * Other than logging the problem we 513204076Spjd * ignore it - nothing smart to do. 514204076Spjd */ 515204076Spjd } 516204076Spjd /* Wait for it to exit. */ 517204076Spjd else if ((pid = waitpid(res->hr_workerpid, 518204076Spjd &status, 0)) != res->hr_workerpid) { 519207372Spjd /* We can only log the problem. */ 520204076Spjd pjdlog_errno(LOG_ERR, 521204076Spjd "Waiting for worker process (pid=%u) failed", 522204076Spjd (unsigned int)res->hr_workerpid); 523204076Spjd } else { 524207372Spjd child_exit_log(res->hr_workerpid, status); 525204076Spjd } 526213006Spjd child_cleanup(res); 527204076Spjd } else if (res->hr_remotein != NULL) { 528204076Spjd char oaddr[256]; 529204076Spjd 530204076Spjd proto_remote_address(conn, oaddr, sizeof(oaddr)); 531204076Spjd pjdlog_debug(1, 532204076Spjd "Canceling half-open connection from %s on connection from %s.", 533204076Spjd oaddr, raddr); 534204076Spjd proto_close(res->hr_remotein); 535204076Spjd res->hr_remotein = NULL; 536204076Spjd } 537204076Spjd } 538204076Spjd 539204076Spjd /* 540204076Spjd * Checks and cleanups are done. 541204076Spjd */ 542204076Spjd 543204076Spjd if (token == NULL) { 544204076Spjd arc4random_buf(res->hr_token, sizeof(res->hr_token)); 545204076Spjd nvout = nv_alloc(); 546204076Spjd nv_add_uint8_array(nvout, res->hr_token, 547204076Spjd sizeof(res->hr_token), "token"); 548204076Spjd if (nv_error(nvout) != 0) { 549204076Spjd pjdlog_common(LOG_ERR, 0, nv_error(nvout), 550204076Spjd "Unable to prepare return header for %s", raddr); 551204076Spjd nv_add_stringf(nverr, "errmsg", 552204076Spjd "Remote node was unable to prepare return header: %s.", 553204076Spjd strerror(nv_error(nvout))); 554204076Spjd goto fail; 555204076Spjd } 556204076Spjd if (hast_proto_send(NULL, conn, nvout, NULL, 0) < 0) { 557204076Spjd int error = errno; 558204076Spjd 559204076Spjd pjdlog_errno(LOG_ERR, "Unable to send response to %s", 560204076Spjd raddr); 561204076Spjd nv_add_stringf(nverr, "errmsg", 562204076Spjd "Remote node was unable to send response: %s.", 563204076Spjd strerror(error)); 564204076Spjd goto fail; 565204076Spjd } 566204076Spjd res->hr_remotein = conn; 567204076Spjd pjdlog_debug(1, "Incoming connection from %s configured.", 568204076Spjd raddr); 569204076Spjd } else { 570204076Spjd res->hr_remoteout = conn; 571204076Spjd pjdlog_debug(1, "Outgoing connection to %s configured.", raddr); 572204076Spjd hastd_secondary(res, nvin); 573204076Spjd } 574204076Spjd nv_free(nvin); 575204076Spjd nv_free(nvout); 576204076Spjd nv_free(nverr); 577204076Spjd pjdlog_prefix_set("%s", ""); 578204076Spjd return; 579204076Spjdfail: 580204076Spjd if (nv_error(nverr) != 0) { 581204076Spjd pjdlog_common(LOG_ERR, 0, nv_error(nverr), 582204076Spjd "Unable to prepare error header for %s", raddr); 583204076Spjd goto close; 584204076Spjd } 585204076Spjd if (hast_proto_send(NULL, conn, nverr, NULL, 0) < 0) { 586204076Spjd pjdlog_errno(LOG_ERR, "Unable to send error to %s", raddr); 587204076Spjd goto close; 588204076Spjd } 589204076Spjdclose: 590204076Spjd if (nvin != NULL) 591204076Spjd nv_free(nvin); 592204076Spjd if (nvout != NULL) 593204076Spjd nv_free(nvout); 594204076Spjd if (nverr != NULL) 595204076Spjd nv_free(nverr); 596204076Spjd proto_close(conn); 597204076Spjd pjdlog_prefix_set("%s", ""); 598204076Spjd} 599204076Spjd 600204076Spjdstatic void 601204076Spjdmain_loop(void) 602204076Spjd{ 603212038Spjd struct hast_resource *res; 604213009Spjd struct timeval seltimeout; 605213009Spjd struct timespec sigtimeout; 606213009Spjd int fd, maxfd, ret, signo; 607213009Spjd sigset_t mask; 608212037Spjd fd_set rfds; 609204076Spjd 610213009Spjd seltimeout.tv_sec = REPORT_INTERVAL; 611213009Spjd seltimeout.tv_usec = 0; 612213009Spjd sigtimeout.tv_sec = 0; 613213009Spjd sigtimeout.tv_nsec = 0; 614211977Spjd 615213009Spjd PJDLOG_VERIFY(sigemptyset(&mask) == 0); 616213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGHUP) == 0); 617213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGINT) == 0); 618213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGTERM) == 0); 619213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGCHLD) == 0); 620213009Spjd 621204076Spjd for (;;) { 622213009Spjd while ((signo = sigtimedwait(&mask, NULL, &sigtimeout)) != -1) { 623213009Spjd switch (signo) { 624213009Spjd case SIGINT: 625213009Spjd case SIGTERM: 626213009Spjd sigexit_received = true; 627213009Spjd terminate_workers(); 628213009Spjd exit(EX_OK); 629213009Spjd break; 630213009Spjd case SIGCHLD: 631213009Spjd child_exit(); 632213009Spjd break; 633213009Spjd case SIGHUP: 634213009Spjd hastd_reload(); 635213009Spjd break; 636213009Spjd default: 637213009Spjd assert(!"invalid condition"); 638213009Spjd } 639211899Spjd } 640204076Spjd 641209177Spjd /* Setup descriptors for select(2). */ 642204076Spjd FD_ZERO(&rfds); 643212038Spjd maxfd = fd = proto_descriptor(cfg->hc_controlconn); 644213008Spjd assert(fd >= 0); 645212038Spjd FD_SET(fd, &rfds); 646212038Spjd fd = proto_descriptor(cfg->hc_listenconn); 647213008Spjd assert(fd >= 0); 648212038Spjd FD_SET(fd, &rfds); 649212038Spjd maxfd = fd > maxfd ? fd : maxfd; 650212038Spjd TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 651212038Spjd if (res->hr_event == NULL) 652212038Spjd continue; 653212038Spjd fd = proto_descriptor(res->hr_event); 654213008Spjd assert(fd >= 0); 655212038Spjd FD_SET(fd, &rfds); 656212038Spjd maxfd = fd > maxfd ? fd : maxfd; 657212038Spjd } 658204076Spjd 659213008Spjd assert(maxfd + 1 <= (int)FD_SETSIZE); 660213009Spjd ret = select(maxfd + 1, &rfds, NULL, NULL, &seltimeout); 661211977Spjd if (ret == 0) 662213429Spjd hook_check(); 663211977Spjd else if (ret == -1) { 664204076Spjd if (errno == EINTR) 665204076Spjd continue; 666204076Spjd KEEP_ERRNO((void)pidfile_remove(pfh)); 667204076Spjd pjdlog_exit(EX_OSERR, "select() failed"); 668204076Spjd } 669204076Spjd 670212038Spjd if (FD_ISSET(proto_descriptor(cfg->hc_controlconn), &rfds)) 671204076Spjd control_handle(cfg); 672212038Spjd if (FD_ISSET(proto_descriptor(cfg->hc_listenconn), &rfds)) 673204076Spjd listen_accept(); 674212038Spjd TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 675212038Spjd if (res->hr_event == NULL) 676212038Spjd continue; 677212038Spjd if (FD_ISSET(proto_descriptor(res->hr_event), &rfds)) { 678212038Spjd if (event_recv(res) == 0) 679212038Spjd continue; 680212038Spjd /* The worker process exited? */ 681212038Spjd proto_close(res->hr_event); 682212038Spjd res->hr_event = NULL; 683212038Spjd } 684212038Spjd } 685204076Spjd } 686204076Spjd} 687204076Spjd 688213428Spjdstatic void 689213428Spjddummy_sighandler(int sig __unused) 690213428Spjd{ 691213428Spjd /* Nothing to do. */ 692213428Spjd} 693213428Spjd 694204076Spjdint 695204076Spjdmain(int argc, char *argv[]) 696204076Spjd{ 697204076Spjd const char *pidfile; 698204076Spjd pid_t otherpid; 699204076Spjd bool foreground; 700204076Spjd int debuglevel; 701213009Spjd sigset_t mask; 702204076Spjd 703204076Spjd g_gate_load(); 704204076Spjd 705204076Spjd foreground = false; 706204076Spjd debuglevel = 0; 707204076Spjd pidfile = HASTD_PIDFILE; 708204076Spjd 709204076Spjd for (;;) { 710204076Spjd int ch; 711204076Spjd 712204076Spjd ch = getopt(argc, argv, "c:dFhP:"); 713204076Spjd if (ch == -1) 714204076Spjd break; 715204076Spjd switch (ch) { 716204076Spjd case 'c': 717204076Spjd cfgpath = optarg; 718204076Spjd break; 719204076Spjd case 'd': 720204076Spjd debuglevel++; 721204076Spjd break; 722204076Spjd case 'F': 723204076Spjd foreground = true; 724204076Spjd break; 725204076Spjd case 'P': 726204076Spjd pidfile = optarg; 727204076Spjd break; 728204076Spjd case 'h': 729204076Spjd default: 730204076Spjd usage(); 731204076Spjd } 732204076Spjd } 733204076Spjd argc -= optind; 734204076Spjd argv += optind; 735204076Spjd 736204076Spjd pjdlog_debug_set(debuglevel); 737204076Spjd 738204076Spjd pfh = pidfile_open(pidfile, 0600, &otherpid); 739204076Spjd if (pfh == NULL) { 740204076Spjd if (errno == EEXIST) { 741204076Spjd pjdlog_exitx(EX_TEMPFAIL, 742204076Spjd "Another hastd is already running, pid: %jd.", 743204076Spjd (intmax_t)otherpid); 744204076Spjd } 745204076Spjd /* If we cannot create pidfile from other reasons, only warn. */ 746210879Spjd pjdlog_errno(LOG_WARNING, "Unable to open or create pidfile"); 747204076Spjd } 748204076Spjd 749210883Spjd cfg = yy_config_parse(cfgpath, true); 750204076Spjd assert(cfg != NULL); 751204076Spjd 752213428Spjd /* 753213428Spjd * Because SIGCHLD is ignored by default, setup dummy handler for it, 754213428Spjd * so we can mask it. 755213428Spjd */ 756213428Spjd PJDLOG_VERIFY(signal(SIGCHLD, dummy_sighandler) != SIG_ERR); 757213009Spjd PJDLOG_VERIFY(sigemptyset(&mask) == 0); 758213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGHUP) == 0); 759213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGINT) == 0); 760213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGTERM) == 0); 761213009Spjd PJDLOG_VERIFY(sigaddset(&mask, SIGCHLD) == 0); 762213009Spjd PJDLOG_VERIFY(sigprocmask(SIG_SETMASK, &mask, NULL) == 0); 763204076Spjd 764204076Spjd /* Listen on control address. */ 765204076Spjd if (proto_server(cfg->hc_controladdr, &cfg->hc_controlconn) < 0) { 766204076Spjd KEEP_ERRNO((void)pidfile_remove(pfh)); 767204076Spjd pjdlog_exit(EX_OSERR, "Unable to listen on control address %s", 768204076Spjd cfg->hc_controladdr); 769204076Spjd } 770204076Spjd /* Listen for remote connections. */ 771204076Spjd if (proto_server(cfg->hc_listenaddr, &cfg->hc_listenconn) < 0) { 772204076Spjd KEEP_ERRNO((void)pidfile_remove(pfh)); 773204076Spjd pjdlog_exit(EX_OSERR, "Unable to listen on address %s", 774204076Spjd cfg->hc_listenaddr); 775204076Spjd } 776204076Spjd 777204076Spjd if (!foreground) { 778204076Spjd if (daemon(0, 0) < 0) { 779204076Spjd KEEP_ERRNO((void)pidfile_remove(pfh)); 780204076Spjd pjdlog_exit(EX_OSERR, "Unable to daemonize"); 781204076Spjd } 782204076Spjd 783204076Spjd /* Start logging to syslog. */ 784204076Spjd pjdlog_mode_set(PJDLOG_MODE_SYSLOG); 785204076Spjd 786204076Spjd /* Write PID to a file. */ 787204076Spjd if (pidfile_write(pfh) < 0) { 788204076Spjd pjdlog_errno(LOG_WARNING, 789204076Spjd "Unable to write PID to a file"); 790204076Spjd } 791204076Spjd } 792204076Spjd 793211977Spjd hook_init(); 794211977Spjd 795204076Spjd main_loop(); 796204076Spjd 797204076Spjd exit(0); 798204076Spjd} 799