hastd.c revision 207345
1/*- 2 * Copyright (c) 2009-2010 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Pawel Jakub Dawidek under sponsorship from 6 * the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD: head/sbin/hastd/hastd.c 207345 2010-04-28 22:26:30Z pjd $"); 32 33#include <sys/param.h> 34#include <sys/linker.h> 35#include <sys/module.h> 36#include <sys/wait.h> 37 38#include <assert.h> 39#include <err.h> 40#include <errno.h> 41#include <libutil.h> 42#include <signal.h> 43#include <stdbool.h> 44#include <stdio.h> 45#include <stdlib.h> 46#include <string.h> 47#include <sysexits.h> 48#include <unistd.h> 49 50#include <activemap.h> 51#include <pjdlog.h> 52 53#include "control.h" 54#include "hast.h" 55#include "hast_proto.h" 56#include "hastd.h" 57#include "subr.h" 58 59/* Path to configuration file. */ 60static const char *cfgpath = HAST_CONFIG; 61/* Hastd configuration. */ 62static struct hastd_config *cfg; 63/* Was SIGCHLD signal received? */ 64static bool sigchld_received = false; 65/* Was SIGHUP signal received? */ 66static bool sighup_received = false; 67/* Was SIGINT or SIGTERM signal received? */ 68bool sigexit_received = false; 69/* PID file handle. */ 70struct pidfh *pfh; 71 72static void 73usage(void) 74{ 75 76 errx(EX_USAGE, "[-dFh] [-c config] [-P pidfile]"); 77} 78 79static void 80sighandler(int sig) 81{ 82 83 switch (sig) { 84 case SIGCHLD: 85 sigchld_received = true; 86 break; 87 case SIGHUP: 88 sighup_received = true; 89 break; 90 default: 91 assert(!"invalid condition"); 92 } 93} 94 95static void 96g_gate_load(void) 97{ 98 99 if (modfind("g_gate") == -1) { 100 /* Not present in kernel, try loading it. */ 101 if (kldload("geom_gate") == -1 || modfind("g_gate") == -1) { 102 if (errno != EEXIST) { 103 pjdlog_exit(EX_OSERR, 104 "Unable to load geom_gate module"); 105 } 106 } 107 } 108} 109 110static void 111child_exit(void) 112{ 113 struct hast_resource *res; 114 int status; 115 pid_t pid; 116 117 while ((pid = wait3(&status, WNOHANG, NULL)) > 0) { 118 /* Find resource related to the process that just exited. */ 119 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 120 if (pid == res->hr_workerpid) 121 break; 122 } 123 if (res == NULL) { 124 /* 125 * This can happen when new connection arrives and we 126 * cancel child responsible for the old one. 127 */ 128 continue; 129 } 130 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, 131 role2str(res->hr_role)); 132 if (WEXITSTATUS(status) == 0) { 133 pjdlog_debug(1, 134 "Worker process exited gracefully (pid=%u).", 135 (unsigned int)pid); 136 } else { 137 pjdlog_error("Worker process failed (pid=%u, status=%d).", 138 (unsigned int)pid, WEXITSTATUS(status)); 139 } 140 proto_close(res->hr_ctrl); 141 res->hr_workerpid = 0; 142 if (res->hr_role == HAST_ROLE_PRIMARY) { 143 sleep(1); 144 pjdlog_info("Restarting worker process."); 145 hastd_primary(res); 146 } 147 pjdlog_prefix_set("%s", ""); 148 } 149} 150 151static void 152hastd_reload(void) 153{ 154 155 /* TODO */ 156 pjdlog_warning("Configuration reload is not implemented."); 157} 158 159static void 160listen_accept(void) 161{ 162 struct hast_resource *res; 163 struct proto_conn *conn; 164 struct nv *nvin, *nvout, *nverr; 165 const char *resname; 166 const unsigned char *token; 167 char laddr[256], raddr[256]; 168 size_t size; 169 pid_t pid; 170 int status; 171 172 proto_local_address(cfg->hc_listenconn, laddr, sizeof(laddr)); 173 pjdlog_debug(1, "Accepting connection to %s.", laddr); 174 175 if (proto_accept(cfg->hc_listenconn, &conn) < 0) { 176 pjdlog_errno(LOG_ERR, "Unable to accept connection %s", laddr); 177 return; 178 } 179 180 proto_local_address(conn, laddr, sizeof(laddr)); 181 proto_remote_address(conn, raddr, sizeof(raddr)); 182 pjdlog_info("Connection from %s to %s.", laddr, raddr); 183 184 nvin = nvout = nverr = NULL; 185 186 /* 187 * Before receiving any data see if remote host have access to any 188 * resource. 189 */ 190 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 191 if (proto_address_match(conn, res->hr_remoteaddr)) 192 break; 193 } 194 if (res == NULL) { 195 pjdlog_error("Client %s isn't known.", raddr); 196 goto close; 197 } 198 /* Ok, remote host can access at least one resource. */ 199 200 if (hast_proto_recv_hdr(conn, &nvin) < 0) { 201 pjdlog_errno(LOG_ERR, "Unable to receive header from %s", 202 raddr); 203 goto close; 204 } 205 206 resname = nv_get_string(nvin, "resource"); 207 if (resname == NULL) { 208 pjdlog_error("No 'resource' field in the header received from %s.", 209 raddr); 210 goto close; 211 } 212 pjdlog_debug(2, "%s: resource=%s", raddr, resname); 213 token = nv_get_uint8_array(nvin, &size, "token"); 214 /* 215 * NULL token means that this is first conection. 216 */ 217 if (token != NULL && size != sizeof(res->hr_token)) { 218 pjdlog_error("Received token of invalid size from %s (expected %zu, got %zu).", 219 raddr, sizeof(res->hr_token), size); 220 goto close; 221 } 222 223 /* 224 * From now on we want to send errors to the remote node. 225 */ 226 nverr = nv_alloc(); 227 228 /* Find resource related to this connection. */ 229 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) { 230 if (strcmp(resname, res->hr_name) == 0) 231 break; 232 } 233 /* Have we found the resource? */ 234 if (res == NULL) { 235 pjdlog_error("No resource '%s' as requested by %s.", 236 resname, raddr); 237 nv_add_stringf(nverr, "errmsg", "Resource not configured."); 238 goto fail; 239 } 240 241 /* Now that we know resource name setup log prefix. */ 242 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role)); 243 244 /* Does the remote host have access to this resource? */ 245 if (!proto_address_match(conn, res->hr_remoteaddr)) { 246 pjdlog_error("Client %s has no access to the resource.", raddr); 247 nv_add_stringf(nverr, "errmsg", "No access to the resource."); 248 goto fail; 249 } 250 /* Is the resource marked as secondary? */ 251 if (res->hr_role != HAST_ROLE_SECONDARY) { 252 pjdlog_error("We act as %s for the resource and not as %s as requested by %s.", 253 role2str(res->hr_role), role2str(HAST_ROLE_SECONDARY), 254 raddr); 255 nv_add_stringf(nverr, "errmsg", 256 "Remote node acts as %s for the resource and not as %s.", 257 role2str(res->hr_role), role2str(HAST_ROLE_SECONDARY)); 258 goto fail; 259 } 260 /* Does token (if exists) match? */ 261 if (token != NULL && memcmp(token, res->hr_token, 262 sizeof(res->hr_token)) != 0) { 263 pjdlog_error("Token received from %s doesn't match.", raddr); 264 nv_add_stringf(nverr, "errmsg", "Toke doesn't match."); 265 goto fail; 266 } 267 /* 268 * If there is no token, but we have half-open connection 269 * (only remotein) or full connection (worker process is running) 270 * we have to cancel those and accept the new connection. 271 */ 272 if (token == NULL) { 273 assert(res->hr_remoteout == NULL); 274 pjdlog_debug(1, "Initial connection from %s.", raddr); 275 if (res->hr_workerpid != 0) { 276 assert(res->hr_remotein == NULL); 277 pjdlog_debug(1, 278 "Worker process exists (pid=%u), stopping it.", 279 (unsigned int)res->hr_workerpid); 280 /* Stop child process. */ 281 if (kill(res->hr_workerpid, SIGINT) < 0) { 282 pjdlog_errno(LOG_ERR, 283 "Unable to stop worker process (pid=%u)", 284 (unsigned int)res->hr_workerpid); 285 /* 286 * Other than logging the problem we 287 * ignore it - nothing smart to do. 288 */ 289 } 290 /* Wait for it to exit. */ 291 else if ((pid = waitpid(res->hr_workerpid, 292 &status, 0)) != res->hr_workerpid) { 293 pjdlog_errno(LOG_ERR, 294 "Waiting for worker process (pid=%u) failed", 295 (unsigned int)res->hr_workerpid); 296 /* See above. */ 297 } else if (WEXITSTATUS(status) != 0) { 298 pjdlog_error("Worker process (pid=%u) exited ungracefully: status=%d.", 299 (unsigned int)res->hr_workerpid, 300 WEXITSTATUS(status)); 301 /* See above. */ 302 } else { 303 pjdlog_debug(1, 304 "Worker process (pid=%u) exited gracefully.", 305 (unsigned int)res->hr_workerpid); 306 } 307 res->hr_workerpid = 0; 308 } else if (res->hr_remotein != NULL) { 309 char oaddr[256]; 310 311 proto_remote_address(conn, oaddr, sizeof(oaddr)); 312 pjdlog_debug(1, 313 "Canceling half-open connection from %s on connection from %s.", 314 oaddr, raddr); 315 proto_close(res->hr_remotein); 316 res->hr_remotein = NULL; 317 } 318 } 319 320 /* 321 * Checks and cleanups are done. 322 */ 323 324 if (token == NULL) { 325 arc4random_buf(res->hr_token, sizeof(res->hr_token)); 326 nvout = nv_alloc(); 327 nv_add_uint8_array(nvout, res->hr_token, 328 sizeof(res->hr_token), "token"); 329 if (nv_error(nvout) != 0) { 330 pjdlog_common(LOG_ERR, 0, nv_error(nvout), 331 "Unable to prepare return header for %s", raddr); 332 nv_add_stringf(nverr, "errmsg", 333 "Remote node was unable to prepare return header: %s.", 334 strerror(nv_error(nvout))); 335 goto fail; 336 } 337 if (hast_proto_send(NULL, conn, nvout, NULL, 0) < 0) { 338 int error = errno; 339 340 pjdlog_errno(LOG_ERR, "Unable to send response to %s", 341 raddr); 342 nv_add_stringf(nverr, "errmsg", 343 "Remote node was unable to send response: %s.", 344 strerror(error)); 345 goto fail; 346 } 347 res->hr_remotein = conn; 348 pjdlog_debug(1, "Incoming connection from %s configured.", 349 raddr); 350 } else { 351 res->hr_remoteout = conn; 352 pjdlog_debug(1, "Outgoing connection to %s configured.", raddr); 353 hastd_secondary(res, nvin); 354 } 355 nv_free(nvin); 356 nv_free(nvout); 357 nv_free(nverr); 358 pjdlog_prefix_set("%s", ""); 359 return; 360fail: 361 if (nv_error(nverr) != 0) { 362 pjdlog_common(LOG_ERR, 0, nv_error(nverr), 363 "Unable to prepare error header for %s", raddr); 364 goto close; 365 } 366 if (hast_proto_send(NULL, conn, nverr, NULL, 0) < 0) { 367 pjdlog_errno(LOG_ERR, "Unable to send error to %s", raddr); 368 goto close; 369 } 370close: 371 if (nvin != NULL) 372 nv_free(nvin); 373 if (nvout != NULL) 374 nv_free(nvout); 375 if (nverr != NULL) 376 nv_free(nverr); 377 proto_close(conn); 378 pjdlog_prefix_set("%s", ""); 379} 380 381static void 382main_loop(void) 383{ 384 fd_set rfds, wfds; 385 int fd, maxfd, ret; 386 387 for (;;) { 388 if (sigchld_received) { 389 sigchld_received = false; 390 child_exit(); 391 } 392 if (sighup_received) { 393 sighup_received = false; 394 hastd_reload(); 395 } 396 397 maxfd = 0; 398 FD_ZERO(&rfds); 399 FD_ZERO(&wfds); 400 401 /* Setup descriptors for select(2). */ 402#define SETUP_FD(conn) do { \ 403 fd = proto_descriptor(conn); \ 404 if (fd >= 0) { \ 405 maxfd = fd > maxfd ? fd : maxfd; \ 406 FD_SET(fd, &rfds); \ 407 FD_SET(fd, &wfds); \ 408 } \ 409} while (0) 410 SETUP_FD(cfg->hc_controlconn); 411 SETUP_FD(cfg->hc_listenconn); 412#undef SETUP_FD 413 414 ret = select(maxfd + 1, &rfds, &wfds, NULL, NULL); 415 if (ret == -1) { 416 if (errno == EINTR) 417 continue; 418 KEEP_ERRNO((void)pidfile_remove(pfh)); 419 pjdlog_exit(EX_OSERR, "select() failed"); 420 } 421 422#define ISSET_FD(conn) \ 423 (FD_ISSET((fd = proto_descriptor(conn)), &rfds) || FD_ISSET(fd, &wfds)) 424 if (ISSET_FD(cfg->hc_controlconn)) 425 control_handle(cfg); 426 if (ISSET_FD(cfg->hc_listenconn)) 427 listen_accept(); 428#undef ISSET_FD 429 } 430} 431 432int 433main(int argc, char *argv[]) 434{ 435 const char *pidfile; 436 pid_t otherpid; 437 bool foreground; 438 int debuglevel; 439 440 g_gate_load(); 441 442 foreground = false; 443 debuglevel = 0; 444 pidfile = HASTD_PIDFILE; 445 446 for (;;) { 447 int ch; 448 449 ch = getopt(argc, argv, "c:dFhP:"); 450 if (ch == -1) 451 break; 452 switch (ch) { 453 case 'c': 454 cfgpath = optarg; 455 break; 456 case 'd': 457 debuglevel++; 458 break; 459 case 'F': 460 foreground = true; 461 break; 462 case 'P': 463 pidfile = optarg; 464 break; 465 case 'h': 466 default: 467 usage(); 468 } 469 } 470 argc -= optind; 471 argv += optind; 472 473 pjdlog_debug_set(debuglevel); 474 475 pfh = pidfile_open(pidfile, 0600, &otherpid); 476 if (pfh == NULL) { 477 if (errno == EEXIST) { 478 pjdlog_exitx(EX_TEMPFAIL, 479 "Another hastd is already running, pid: %jd.", 480 (intmax_t)otherpid); 481 } 482 /* If we cannot create pidfile from other reasons, only warn. */ 483 pjdlog_errno(LOG_WARNING, "Cannot open or create pidfile"); 484 } 485 486 cfg = yy_config_parse(cfgpath); 487 assert(cfg != NULL); 488 489 signal(SIGHUP, sighandler); 490 signal(SIGCHLD, sighandler); 491 492 /* Listen on control address. */ 493 if (proto_server(cfg->hc_controladdr, &cfg->hc_controlconn) < 0) { 494 KEEP_ERRNO((void)pidfile_remove(pfh)); 495 pjdlog_exit(EX_OSERR, "Unable to listen on control address %s", 496 cfg->hc_controladdr); 497 } 498 /* Listen for remote connections. */ 499 if (proto_server(cfg->hc_listenaddr, &cfg->hc_listenconn) < 0) { 500 KEEP_ERRNO((void)pidfile_remove(pfh)); 501 pjdlog_exit(EX_OSERR, "Unable to listen on address %s", 502 cfg->hc_listenaddr); 503 } 504 505 if (!foreground) { 506 if (daemon(0, 0) < 0) { 507 KEEP_ERRNO((void)pidfile_remove(pfh)); 508 pjdlog_exit(EX_OSERR, "Unable to daemonize"); 509 } 510 511 /* Start logging to syslog. */ 512 pjdlog_mode_set(PJDLOG_MODE_SYSLOG); 513 514 /* Write PID to a file. */ 515 if (pidfile_write(pfh) < 0) { 516 pjdlog_errno(LOG_WARNING, 517 "Unable to write PID to a file"); 518 } 519 } 520 521 main_loop(); 522 523 exit(0); 524} 525