1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2015 Nexenta Systems, Inc. All rights reserved. 25 * Copyright (c) 2016, Intel Corporation. 26 */ 27 28#include <stddef.h> 29#include <string.h> 30#include <strings.h> 31#include <libuutil.h> 32#include <libzfs.h> 33#include <sys/types.h> 34#include <sys/time.h> 35#include <sys/fs/zfs.h> 36#include <sys/fm/protocol.h> 37#include <sys/fm/fs/zfs.h> 38 39#include "zfs_agents.h" 40#include "fmd_api.h" 41 42/* 43 * Our serd engines are named 'zfs_<pool_guid>_<vdev_guid>_{checksum,io}'. This 44 * #define reserves enough space for two 64-bit hex values plus the length of 45 * the longest string. 46 */ 47#define MAX_SERDLEN (16 * 2 + sizeof ("zfs___checksum")) 48 49/* 50 * On-disk case structure. This must maintain backwards compatibility with 51 * previous versions of the DE. By default, any members appended to the end 52 * will be filled with zeros if they don't exist in a previous version. 53 */ 54typedef struct zfs_case_data { 55 uint64_t zc_version; 56 uint64_t zc_ena; 57 uint64_t zc_pool_guid; 58 uint64_t zc_vdev_guid; 59 int zc_pool_state; 60 char zc_serd_checksum[MAX_SERDLEN]; 61 char zc_serd_io[MAX_SERDLEN]; 62 int zc_has_remove_timer; 63} zfs_case_data_t; 64 65/* 66 * Time-of-day 67 */ 68typedef struct er_timeval { 69 uint64_t ertv_sec; 70 uint64_t ertv_nsec; 71} er_timeval_t; 72 73/* 74 * In-core case structure. 75 */ 76typedef struct zfs_case { 77 boolean_t zc_present; 78 uint32_t zc_version; 79 zfs_case_data_t zc_data; 80 fmd_case_t *zc_case; 81 uu_list_node_t zc_node; 82 id_t zc_remove_timer; 83 char *zc_fru; 84 er_timeval_t zc_when; 85} zfs_case_t; 86 87#define CASE_DATA "data" 88#define CASE_FRU "fru" 89#define CASE_DATA_VERSION_INITIAL 1 90#define CASE_DATA_VERSION_SERD 2 91 92typedef struct zfs_de_stats { 93 fmd_stat_t old_drops; 94 fmd_stat_t dev_drops; 95 fmd_stat_t vdev_drops; 96 fmd_stat_t import_drops; 97 fmd_stat_t resource_drops; 98} zfs_de_stats_t; 99 100zfs_de_stats_t zfs_stats = { 101 { "old_drops", FMD_TYPE_UINT64, "ereports dropped (from before load)" }, 102 { "dev_drops", FMD_TYPE_UINT64, "ereports dropped (dev during open)"}, 103 { "vdev_drops", FMD_TYPE_UINT64, "ereports dropped (weird vdev types)"}, 104 { "import_drops", FMD_TYPE_UINT64, "ereports dropped (during import)" }, 105 { "resource_drops", FMD_TYPE_UINT64, "resource related ereports" } 106}; 107 108static hrtime_t zfs_remove_timeout; 109 110uu_list_pool_t *zfs_case_pool; 111uu_list_t *zfs_cases; 112 113#define ZFS_MAKE_RSRC(type) \ 114 FM_RSRC_CLASS "." ZFS_ERROR_CLASS "." type 115#define ZFS_MAKE_EREPORT(type) \ 116 FM_EREPORT_CLASS "." ZFS_ERROR_CLASS "." type 117 118/* 119 * Write out the persistent representation of an active case. 120 */ 121static void 122zfs_case_serialize(fmd_hdl_t *hdl, zfs_case_t *zcp) 123{ 124 zcp->zc_data.zc_version = CASE_DATA_VERSION_SERD; 125} 126 127/* 128 * Read back the persistent representation of an active case. 129 */ 130static zfs_case_t * 131zfs_case_unserialize(fmd_hdl_t *hdl, fmd_case_t *cp) 132{ 133 zfs_case_t *zcp; 134 135 zcp = fmd_hdl_zalloc(hdl, sizeof (zfs_case_t), FMD_SLEEP); 136 zcp->zc_case = cp; 137 138 fmd_buf_read(hdl, cp, CASE_DATA, &zcp->zc_data, 139 sizeof (zcp->zc_data)); 140 141 if (zcp->zc_data.zc_version > CASE_DATA_VERSION_SERD) { 142 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t)); 143 return (NULL); 144 } 145 146 /* 147 * fmd_buf_read() will have already zeroed out the remainder of the 148 * buffer, so we don't have to do anything special if the version 149 * doesn't include the SERD engine name. 150 */ 151 152 if (zcp->zc_data.zc_has_remove_timer) 153 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, 154 NULL, zfs_remove_timeout); 155 156 uu_list_node_init(zcp, &zcp->zc_node, zfs_case_pool); 157 (void) uu_list_insert_before(zfs_cases, NULL, zcp); 158 159 fmd_case_setspecific(hdl, cp, zcp); 160 161 return (zcp); 162} 163 164/* 165 * Iterate over any active cases. If any cases are associated with a pool or 166 * vdev which is no longer present on the system, close the associated case. 167 */ 168static void 169zfs_mark_vdev(uint64_t pool_guid, nvlist_t *vd, er_timeval_t *loaded) 170{ 171 uint64_t vdev_guid = 0; 172 uint_t c, children; 173 nvlist_t **child; 174 zfs_case_t *zcp; 175 176 (void) nvlist_lookup_uint64(vd, ZPOOL_CONFIG_GUID, &vdev_guid); 177 178 /* 179 * Mark any cases associated with this (pool, vdev) pair. 180 */ 181 for (zcp = uu_list_first(zfs_cases); zcp != NULL; 182 zcp = uu_list_next(zfs_cases, zcp)) { 183 if (zcp->zc_data.zc_pool_guid == pool_guid && 184 zcp->zc_data.zc_vdev_guid == vdev_guid) { 185 zcp->zc_present = B_TRUE; 186 zcp->zc_when = *loaded; 187 } 188 } 189 190 /* 191 * Iterate over all children. 192 */ 193 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_CHILDREN, &child, 194 &children) == 0) { 195 for (c = 0; c < children; c++) 196 zfs_mark_vdev(pool_guid, child[c], loaded); 197 } 198 199 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_L2CACHE, &child, 200 &children) == 0) { 201 for (c = 0; c < children; c++) 202 zfs_mark_vdev(pool_guid, child[c], loaded); 203 } 204 205 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_SPARES, &child, 206 &children) == 0) { 207 for (c = 0; c < children; c++) 208 zfs_mark_vdev(pool_guid, child[c], loaded); 209 } 210} 211 212/*ARGSUSED*/ 213static int 214zfs_mark_pool(zpool_handle_t *zhp, void *unused) 215{ 216 zfs_case_t *zcp; 217 uint64_t pool_guid; 218 uint64_t *tod; 219 er_timeval_t loaded = { 0 }; 220 nvlist_t *config, *vd; 221 uint_t nelem = 0; 222 int ret; 223 224 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL); 225 /* 226 * Mark any cases associated with just this pool. 227 */ 228 for (zcp = uu_list_first(zfs_cases); zcp != NULL; 229 zcp = uu_list_next(zfs_cases, zcp)) { 230 if (zcp->zc_data.zc_pool_guid == pool_guid && 231 zcp->zc_data.zc_vdev_guid == 0) 232 zcp->zc_present = B_TRUE; 233 } 234 235 if ((config = zpool_get_config(zhp, NULL)) == NULL) { 236 zpool_close(zhp); 237 return (-1); 238 } 239 240 (void) nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME, 241 &tod, &nelem); 242 if (nelem == 2) { 243 loaded.ertv_sec = tod[0]; 244 loaded.ertv_nsec = tod[1]; 245 for (zcp = uu_list_first(zfs_cases); zcp != NULL; 246 zcp = uu_list_next(zfs_cases, zcp)) { 247 if (zcp->zc_data.zc_pool_guid == pool_guid && 248 zcp->zc_data.zc_vdev_guid == 0) { 249 zcp->zc_when = loaded; 250 } 251 } 252 } 253 254 ret = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vd); 255 if (ret) { 256 zpool_close(zhp); 257 return (-1); 258 } 259 260 zfs_mark_vdev(pool_guid, vd, &loaded); 261 262 zpool_close(zhp); 263 264 return (0); 265} 266 267struct load_time_arg { 268 uint64_t lt_guid; 269 er_timeval_t *lt_time; 270 boolean_t lt_found; 271}; 272 273static int 274zpool_find_load_time(zpool_handle_t *zhp, void *arg) 275{ 276 struct load_time_arg *lta = arg; 277 uint64_t pool_guid; 278 uint64_t *tod; 279 nvlist_t *config; 280 uint_t nelem; 281 282 if (lta->lt_found) { 283 zpool_close(zhp); 284 return (0); 285 } 286 287 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL); 288 if (pool_guid != lta->lt_guid) { 289 zpool_close(zhp); 290 return (0); 291 } 292 293 if ((config = zpool_get_config(zhp, NULL)) == NULL) { 294 zpool_close(zhp); 295 return (-1); 296 } 297 298 if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME, 299 &tod, &nelem) == 0 && nelem == 2) { 300 lta->lt_found = B_TRUE; 301 lta->lt_time->ertv_sec = tod[0]; 302 lta->lt_time->ertv_nsec = tod[1]; 303 } 304 305 zpool_close(zhp); 306 307 return (0); 308} 309 310static void 311zfs_purge_cases(fmd_hdl_t *hdl) 312{ 313 zfs_case_t *zcp; 314 uu_list_walk_t *walk; 315 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl); 316 317 /* 318 * There is no way to open a pool by GUID, or lookup a vdev by GUID. No 319 * matter what we do, we're going to have to stomach an O(vdevs * cases) 320 * algorithm. In reality, both quantities are likely so small that 321 * neither will matter. Given that iterating over pools is more 322 * expensive than iterating over the in-memory case list, we opt for a 323 * 'present' flag in each case that starts off cleared. We then iterate 324 * over all pools, marking those that are still present, and removing 325 * those that aren't found. 326 * 327 * Note that we could also construct an FMRI and rely on 328 * fmd_nvl_fmri_present(), but this would end up doing the same search. 329 */ 330 331 /* 332 * Mark the cases as not present. 333 */ 334 for (zcp = uu_list_first(zfs_cases); zcp != NULL; 335 zcp = uu_list_next(zfs_cases, zcp)) 336 zcp->zc_present = B_FALSE; 337 338 /* 339 * Iterate over all pools and mark the pools and vdevs found. If this 340 * fails (most probably because we're out of memory), then don't close 341 * any of the cases and we cannot be sure they are accurate. 342 */ 343 if (zpool_iter(zhdl, zfs_mark_pool, NULL) != 0) 344 return; 345 346 /* 347 * Remove those cases which were not found. 348 */ 349 walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST); 350 while ((zcp = uu_list_walk_next(walk)) != NULL) { 351 if (!zcp->zc_present) 352 fmd_case_close(hdl, zcp->zc_case); 353 } 354 uu_list_walk_end(walk); 355} 356 357/* 358 * Construct the name of a serd engine given the pool/vdev GUID and type (io or 359 * checksum). 360 */ 361static void 362zfs_serd_name(char *buf, uint64_t pool_guid, uint64_t vdev_guid, 363 const char *type) 364{ 365 (void) snprintf(buf, MAX_SERDLEN, "zfs_%llx_%llx_%s", 366 (long long unsigned int)pool_guid, 367 (long long unsigned int)vdev_guid, type); 368} 369 370/* 371 * Solve a given ZFS case. This first checks to make sure the diagnosis is 372 * still valid, as well as cleaning up any pending timer associated with the 373 * case. 374 */ 375static void 376zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname, 377 boolean_t checkunusable) 378{ 379 nvlist_t *detector, *fault; 380 boolean_t serialize; 381 nvlist_t *fru = NULL; 382 fmd_hdl_debug(hdl, "solving fault '%s'", faultname); 383 384 /* 385 * Construct the detector from the case data. The detector is in the 386 * ZFS scheme, and is either the pool or the vdev, depending on whether 387 * this is a vdev or pool fault. 388 */ 389 detector = fmd_nvl_alloc(hdl, FMD_SLEEP); 390 391 (void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0); 392 (void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS); 393 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL, 394 zcp->zc_data.zc_pool_guid); 395 if (zcp->zc_data.zc_vdev_guid != 0) { 396 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV, 397 zcp->zc_data.zc_vdev_guid); 398 } 399 400 fault = fmd_nvl_create_fault(hdl, faultname, 100, detector, 401 fru, detector); 402 fmd_case_add_suspect(hdl, zcp->zc_case, fault); 403 404 nvlist_free(fru); 405 406 fmd_case_solve(hdl, zcp->zc_case); 407 408 serialize = B_FALSE; 409 if (zcp->zc_data.zc_has_remove_timer) { 410 fmd_timer_remove(hdl, zcp->zc_remove_timer); 411 zcp->zc_data.zc_has_remove_timer = 0; 412 serialize = B_TRUE; 413 } 414 if (serialize) 415 zfs_case_serialize(hdl, zcp); 416 417 nvlist_free(detector); 418} 419 420static boolean_t 421timeval_earlier(er_timeval_t *a, er_timeval_t *b) 422{ 423 return (a->ertv_sec < b->ertv_sec || 424 (a->ertv_sec == b->ertv_sec && a->ertv_nsec < b->ertv_nsec)); 425} 426 427/*ARGSUSED*/ 428static void 429zfs_ereport_when(fmd_hdl_t *hdl, nvlist_t *nvl, er_timeval_t *when) 430{ 431 int64_t *tod; 432 uint_t nelem; 433 434 if (nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tod, 435 &nelem) == 0 && nelem == 2) { 436 when->ertv_sec = tod[0]; 437 when->ertv_nsec = tod[1]; 438 } else { 439 when->ertv_sec = when->ertv_nsec = UINT64_MAX; 440 } 441} 442 443/* 444 * Main fmd entry point. 445 */ 446/*ARGSUSED*/ 447static void 448zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class) 449{ 450 zfs_case_t *zcp, *dcp; 451 int32_t pool_state; 452 uint64_t ena, pool_guid, vdev_guid; 453 er_timeval_t pool_load; 454 er_timeval_t er_when; 455 nvlist_t *detector; 456 boolean_t pool_found = B_FALSE; 457 boolean_t isresource; 458 char *type; 459 460 /* 461 * We subscribe to notifications for vdev or pool removal. In these 462 * cases, there may be cases that no longer apply. Purge any cases 463 * that no longer apply. 464 */ 465 if (fmd_nvl_class_match(hdl, nvl, "sysevent.fs.zfs.*")) { 466 fmd_hdl_debug(hdl, "purging orphaned cases from %s", 467 strrchr(class, '.') + 1); 468 zfs_purge_cases(hdl); 469 zfs_stats.resource_drops.fmds_value.ui64++; 470 return; 471 } 472 473 isresource = fmd_nvl_class_match(hdl, nvl, "resource.fs.zfs.*"); 474 475 if (isresource) { 476 /* 477 * For resources, we don't have a normal payload. 478 */ 479 if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, 480 &vdev_guid) != 0) 481 pool_state = SPA_LOAD_OPEN; 482 else 483 pool_state = SPA_LOAD_NONE; 484 detector = NULL; 485 } else { 486 (void) nvlist_lookup_nvlist(nvl, 487 FM_EREPORT_DETECTOR, &detector); 488 (void) nvlist_lookup_int32(nvl, 489 FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, &pool_state); 490 } 491 492 /* 493 * We also ignore all ereports generated during an import of a pool, 494 * since the only possible fault (.pool) would result in import failure, 495 * and hence no persistent fault. Some day we may want to do something 496 * with these ereports, so we continue generating them internally. 497 */ 498 if (pool_state == SPA_LOAD_IMPORT) { 499 zfs_stats.import_drops.fmds_value.ui64++; 500 fmd_hdl_debug(hdl, "ignoring '%s' during import", class); 501 return; 502 } 503 504 /* 505 * Device I/O errors are ignored during pool open. 506 */ 507 if (pool_state == SPA_LOAD_OPEN && 508 (fmd_nvl_class_match(hdl, nvl, 509 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) || 510 fmd_nvl_class_match(hdl, nvl, 511 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) || 512 fmd_nvl_class_match(hdl, nvl, 513 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE)))) { 514 fmd_hdl_debug(hdl, "ignoring '%s' during pool open", class); 515 zfs_stats.dev_drops.fmds_value.ui64++; 516 return; 517 } 518 519 /* 520 * We ignore ereports for anything except disks and files. 521 */ 522 if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE, 523 &type) == 0) { 524 if (strcmp(type, VDEV_TYPE_DISK) != 0 && 525 strcmp(type, VDEV_TYPE_FILE) != 0) { 526 zfs_stats.vdev_drops.fmds_value.ui64++; 527 return; 528 } 529 } 530 531 /* 532 * Determine if this ereport corresponds to an open case. 533 * Each vdev or pool can have a single case. 534 */ 535 (void) nvlist_lookup_uint64(nvl, 536 FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, &pool_guid); 537 if (nvlist_lookup_uint64(nvl, 538 FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0) 539 vdev_guid = 0; 540 if (nvlist_lookup_uint64(nvl, FM_EREPORT_ENA, &ena) != 0) 541 ena = 0; 542 543 zfs_ereport_when(hdl, nvl, &er_when); 544 545 for (zcp = uu_list_first(zfs_cases); zcp != NULL; 546 zcp = uu_list_next(zfs_cases, zcp)) { 547 if (zcp->zc_data.zc_pool_guid == pool_guid) { 548 pool_found = B_TRUE; 549 pool_load = zcp->zc_when; 550 } 551 if (zcp->zc_data.zc_vdev_guid == vdev_guid) 552 break; 553 } 554 555 /* 556 * Avoid falsely accusing a pool of being faulty. Do so by 557 * not replaying ereports that were generated prior to the 558 * current import. If the failure that generated them was 559 * transient because the device was actually removed but we 560 * didn't receive the normal asynchronous notification, we 561 * don't want to mark it as faulted and potentially panic. If 562 * there is still a problem we'd expect not to be able to 563 * import the pool, or that new ereports will be generated 564 * once the pool is used. 565 */ 566 if (pool_found && timeval_earlier(&er_when, &pool_load)) { 567 fmd_hdl_debug(hdl, "ignoring pool %llx, " 568 "ereport time %lld.%lld, pool load time = %lld.%lld", 569 pool_guid, er_when.ertv_sec, er_when.ertv_nsec, 570 pool_load.ertv_sec, pool_load.ertv_nsec); 571 zfs_stats.old_drops.fmds_value.ui64++; 572 return; 573 } 574 575 if (!pool_found) { 576 /* 577 * Haven't yet seen this pool, but same situation 578 * may apply. 579 */ 580 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl); 581 struct load_time_arg la; 582 583 la.lt_guid = pool_guid; 584 la.lt_time = &pool_load; 585 la.lt_found = B_FALSE; 586 587 if (zhdl != NULL && 588 zpool_iter(zhdl, zpool_find_load_time, &la) == 0 && 589 la.lt_found == B_TRUE) { 590 pool_found = B_TRUE; 591 592 if (timeval_earlier(&er_when, &pool_load)) { 593 fmd_hdl_debug(hdl, "ignoring pool %llx, " 594 "ereport time %lld.%lld, " 595 "pool load time = %lld.%lld", 596 pool_guid, er_when.ertv_sec, 597 er_when.ertv_nsec, pool_load.ertv_sec, 598 pool_load.ertv_nsec); 599 zfs_stats.old_drops.fmds_value.ui64++; 600 return; 601 } 602 } 603 } 604 605 if (zcp == NULL) { 606 fmd_case_t *cs; 607 zfs_case_data_t data = { 0 }; 608 609 /* 610 * If this is one of our 'fake' resource ereports, and there is 611 * no case open, simply discard it. 612 */ 613 if (isresource) { 614 zfs_stats.resource_drops.fmds_value.ui64++; 615 fmd_hdl_debug(hdl, "discarding '%s for vdev %llu", 616 class, vdev_guid); 617 return; 618 } 619 620 /* 621 * Skip tracking some ereports 622 */ 623 if (strcmp(class, 624 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DATA)) == 0 || 625 strcmp(class, 626 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CONFIG_CACHE_WRITE)) == 0 || 627 strcmp(class, 628 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY)) == 0) { 629 zfs_stats.resource_drops.fmds_value.ui64++; 630 return; 631 } 632 633 /* 634 * Open a new case. 635 */ 636 cs = fmd_case_open(hdl, NULL); 637 638 fmd_hdl_debug(hdl, "opening case for vdev %llu due to '%s'", 639 vdev_guid, class); 640 641 /* 642 * Initialize the case buffer. To commonize code, we actually 643 * create the buffer with existing data, and then call 644 * zfs_case_unserialize() to instantiate the in-core structure. 645 */ 646 fmd_buf_create(hdl, cs, CASE_DATA, sizeof (zfs_case_data_t)); 647 648 data.zc_version = CASE_DATA_VERSION_SERD; 649 data.zc_ena = ena; 650 data.zc_pool_guid = pool_guid; 651 data.zc_vdev_guid = vdev_guid; 652 data.zc_pool_state = (int)pool_state; 653 654 fmd_buf_write(hdl, cs, CASE_DATA, &data, sizeof (data)); 655 656 zcp = zfs_case_unserialize(hdl, cs); 657 assert(zcp != NULL); 658 if (pool_found) 659 zcp->zc_when = pool_load; 660 } 661 662 if (isresource) { 663 fmd_hdl_debug(hdl, "resource event '%s'", class); 664 665 if (fmd_nvl_class_match(hdl, nvl, 666 ZFS_MAKE_RSRC(FM_RESOURCE_AUTOREPLACE))) { 667 /* 668 * The 'resource.fs.zfs.autoreplace' event indicates 669 * that the pool was loaded with the 'autoreplace' 670 * property set. In this case, any pending device 671 * failures should be ignored, as the asynchronous 672 * autoreplace handling will take care of them. 673 */ 674 fmd_case_close(hdl, zcp->zc_case); 675 } else if (fmd_nvl_class_match(hdl, nvl, 676 ZFS_MAKE_RSRC(FM_RESOURCE_REMOVED))) { 677 /* 678 * The 'resource.fs.zfs.removed' event indicates that 679 * device removal was detected, and the device was 680 * closed asynchronously. If this is the case, we 681 * assume that any recent I/O errors were due to the 682 * device removal, not any fault of the device itself. 683 * We reset the SERD engine, and cancel any pending 684 * timers. 685 */ 686 if (zcp->zc_data.zc_has_remove_timer) { 687 fmd_timer_remove(hdl, zcp->zc_remove_timer); 688 zcp->zc_data.zc_has_remove_timer = 0; 689 zfs_case_serialize(hdl, zcp); 690 } 691 if (zcp->zc_data.zc_serd_io[0] != '\0') 692 fmd_serd_reset(hdl, zcp->zc_data.zc_serd_io); 693 if (zcp->zc_data.zc_serd_checksum[0] != '\0') 694 fmd_serd_reset(hdl, 695 zcp->zc_data.zc_serd_checksum); 696 } else if (fmd_nvl_class_match(hdl, nvl, 697 ZFS_MAKE_RSRC(FM_RESOURCE_STATECHANGE))) { 698 uint64_t state = 0; 699 700 if (zcp != NULL && 701 nvlist_lookup_uint64(nvl, 702 FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE, &state) == 0 && 703 state == VDEV_STATE_HEALTHY) { 704 fmd_hdl_debug(hdl, "closing case after a " 705 "device statechange to healthy"); 706 fmd_case_close(hdl, zcp->zc_case); 707 } 708 } 709 zfs_stats.resource_drops.fmds_value.ui64++; 710 return; 711 } 712 713 /* 714 * Associate the ereport with this case. 715 */ 716 fmd_case_add_ereport(hdl, zcp->zc_case, ep); 717 718 /* 719 * Don't do anything else if this case is already solved. 720 */ 721 if (fmd_case_solved(hdl, zcp->zc_case)) 722 return; 723 724 fmd_hdl_debug(hdl, "error event '%s'", class); 725 726 /* 727 * Determine if we should solve the case and generate a fault. We solve 728 * a case if: 729 * 730 * a. A pool failed to open (ereport.fs.zfs.pool) 731 * b. A device failed to open (ereport.fs.zfs.pool) while a pool 732 * was up and running. 733 * 734 * We may see a series of ereports associated with a pool open, all 735 * chained together by the same ENA. If the pool open succeeds, then 736 * we'll see no further ereports. To detect when a pool open has 737 * succeeded, we associate a timer with the event. When it expires, we 738 * close the case. 739 */ 740 if (fmd_nvl_class_match(hdl, nvl, 741 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_POOL))) { 742 /* 743 * Pool level fault. Before solving the case, go through and 744 * close any open device cases that may be pending. 745 */ 746 for (dcp = uu_list_first(zfs_cases); dcp != NULL; 747 dcp = uu_list_next(zfs_cases, dcp)) { 748 if (dcp->zc_data.zc_pool_guid == 749 zcp->zc_data.zc_pool_guid && 750 dcp->zc_data.zc_vdev_guid != 0) 751 fmd_case_close(hdl, dcp->zc_case); 752 } 753 754 zfs_case_solve(hdl, zcp, "fault.fs.zfs.pool", B_TRUE); 755 } else if (fmd_nvl_class_match(hdl, nvl, 756 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_LOG_REPLAY))) { 757 /* 758 * Pool level fault for reading the intent logs. 759 */ 760 zfs_case_solve(hdl, zcp, "fault.fs.zfs.log_replay", B_TRUE); 761 } else if (fmd_nvl_class_match(hdl, nvl, "ereport.fs.zfs.vdev.*")) { 762 /* 763 * Device fault. 764 */ 765 zfs_case_solve(hdl, zcp, "fault.fs.zfs.device", B_TRUE); 766 } else if (fmd_nvl_class_match(hdl, nvl, 767 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) || 768 fmd_nvl_class_match(hdl, nvl, 769 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) || 770 fmd_nvl_class_match(hdl, nvl, 771 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) || 772 fmd_nvl_class_match(hdl, nvl, 773 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) { 774 char *failmode = NULL; 775 boolean_t checkremove = B_FALSE; 776 777 /* 778 * If this is a checksum or I/O error, then toss it into the 779 * appropriate SERD engine and check to see if it has fired. 780 * Ideally, we want to do something more sophisticated, 781 * (persistent errors for a single data block, etc). For now, 782 * a single SERD engine is sufficient. 783 */ 784 if (fmd_nvl_class_match(hdl, nvl, 785 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) { 786 if (zcp->zc_data.zc_serd_io[0] == '\0') { 787 zfs_serd_name(zcp->zc_data.zc_serd_io, 788 pool_guid, vdev_guid, "io"); 789 fmd_serd_create(hdl, zcp->zc_data.zc_serd_io, 790 fmd_prop_get_int32(hdl, "io_N"), 791 fmd_prop_get_int64(hdl, "io_T")); 792 zfs_case_serialize(hdl, zcp); 793 } 794 if (fmd_serd_record(hdl, zcp->zc_data.zc_serd_io, ep)) 795 checkremove = B_TRUE; 796 } else if (fmd_nvl_class_match(hdl, nvl, 797 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM))) { 798 if (zcp->zc_data.zc_serd_checksum[0] == '\0') { 799 zfs_serd_name(zcp->zc_data.zc_serd_checksum, 800 pool_guid, vdev_guid, "checksum"); 801 fmd_serd_create(hdl, 802 zcp->zc_data.zc_serd_checksum, 803 fmd_prop_get_int32(hdl, "checksum_N"), 804 fmd_prop_get_int64(hdl, "checksum_T")); 805 zfs_case_serialize(hdl, zcp); 806 } 807 if (fmd_serd_record(hdl, 808 zcp->zc_data.zc_serd_checksum, ep)) { 809 zfs_case_solve(hdl, zcp, 810 "fault.fs.zfs.vdev.checksum", B_FALSE); 811 } 812 } else if (fmd_nvl_class_match(hdl, nvl, 813 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) && 814 (nvlist_lookup_string(nvl, 815 FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE, &failmode) == 0) && 816 failmode != NULL) { 817 if (strncmp(failmode, FM_EREPORT_FAILMODE_CONTINUE, 818 strlen(FM_EREPORT_FAILMODE_CONTINUE)) == 0) { 819 zfs_case_solve(hdl, zcp, 820 "fault.fs.zfs.io_failure_continue", 821 B_FALSE); 822 } else if (strncmp(failmode, FM_EREPORT_FAILMODE_WAIT, 823 strlen(FM_EREPORT_FAILMODE_WAIT)) == 0) { 824 zfs_case_solve(hdl, zcp, 825 "fault.fs.zfs.io_failure_wait", B_FALSE); 826 } 827 } else if (fmd_nvl_class_match(hdl, nvl, 828 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) { 829#ifndef __linux__ 830 /* This causes an unexpected fault diagnosis on linux */ 831 checkremove = B_TRUE; 832#endif 833 } 834 835 /* 836 * Because I/O errors may be due to device removal, we postpone 837 * any diagnosis until we're sure that we aren't about to 838 * receive a 'resource.fs.zfs.removed' event. 839 */ 840 if (checkremove) { 841 if (zcp->zc_data.zc_has_remove_timer) 842 fmd_timer_remove(hdl, zcp->zc_remove_timer); 843 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, NULL, 844 zfs_remove_timeout); 845 if (!zcp->zc_data.zc_has_remove_timer) { 846 zcp->zc_data.zc_has_remove_timer = 1; 847 zfs_case_serialize(hdl, zcp); 848 } 849 } 850 } 851} 852 853/* 854 * The timeout is fired when we diagnosed an I/O error, and it was not due to 855 * device removal (which would cause the timeout to be cancelled). 856 */ 857/* ARGSUSED */ 858static void 859zfs_fm_timeout(fmd_hdl_t *hdl, id_t id, void *data) 860{ 861 zfs_case_t *zcp = data; 862 863 if (id == zcp->zc_remove_timer) 864 zfs_case_solve(hdl, zcp, "fault.fs.zfs.vdev.io", B_FALSE); 865} 866 867/* 868 * The specified case has been closed and any case-specific 869 * data structures should be deallocated. 870 */ 871static void 872zfs_fm_close(fmd_hdl_t *hdl, fmd_case_t *cs) 873{ 874 zfs_case_t *zcp = fmd_case_getspecific(hdl, cs); 875 876 if (zcp->zc_data.zc_serd_checksum[0] != '\0') 877 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_checksum); 878 if (zcp->zc_data.zc_serd_io[0] != '\0') 879 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_io); 880 if (zcp->zc_data.zc_has_remove_timer) 881 fmd_timer_remove(hdl, zcp->zc_remove_timer); 882 883 uu_list_remove(zfs_cases, zcp); 884 uu_list_node_fini(zcp, &zcp->zc_node, zfs_case_pool); 885 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t)); 886} 887 888/* 889 * We use the fmd gc entry point to look for old cases that no longer apply. 890 * This allows us to keep our set of case data small in a long running system. 891 */ 892static void 893zfs_fm_gc(fmd_hdl_t *hdl) 894{ 895 zfs_purge_cases(hdl); 896} 897 898static const fmd_hdl_ops_t fmd_ops = { 899 zfs_fm_recv, /* fmdo_recv */ 900 zfs_fm_timeout, /* fmdo_timeout */ 901 zfs_fm_close, /* fmdo_close */ 902 NULL, /* fmdo_stats */ 903 zfs_fm_gc, /* fmdo_gc */ 904}; 905 906static const fmd_prop_t fmd_props[] = { 907 { "checksum_N", FMD_TYPE_UINT32, "10" }, 908 { "checksum_T", FMD_TYPE_TIME, "10min" }, 909 { "io_N", FMD_TYPE_UINT32, "10" }, 910 { "io_T", FMD_TYPE_TIME, "10min" }, 911 { "remove_timeout", FMD_TYPE_TIME, "15sec" }, 912 { NULL, 0, NULL } 913}; 914 915static const fmd_hdl_info_t fmd_info = { 916 "ZFS Diagnosis Engine", "1.0", &fmd_ops, fmd_props 917}; 918 919void 920_zfs_diagnosis_init(fmd_hdl_t *hdl) 921{ 922 libzfs_handle_t *zhdl; 923 924 if ((zhdl = libzfs_init()) == NULL) 925 return; 926 927 if ((zfs_case_pool = uu_list_pool_create("zfs_case_pool", 928 sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node), 929 NULL, UU_LIST_POOL_DEBUG)) == NULL) { 930 libzfs_fini(zhdl); 931 return; 932 } 933 934 if ((zfs_cases = uu_list_create(zfs_case_pool, NULL, 935 UU_LIST_DEBUG)) == NULL) { 936 uu_list_pool_destroy(zfs_case_pool); 937 libzfs_fini(zhdl); 938 return; 939 } 940 941 if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) { 942 uu_list_destroy(zfs_cases); 943 uu_list_pool_destroy(zfs_case_pool); 944 libzfs_fini(zhdl); 945 return; 946 } 947 948 fmd_hdl_setspecific(hdl, zhdl); 949 950 (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (zfs_stats) / 951 sizeof (fmd_stat_t), (fmd_stat_t *)&zfs_stats); 952 953 zfs_remove_timeout = fmd_prop_get_int64(hdl, "remove_timeout"); 954} 955 956void 957_zfs_diagnosis_fini(fmd_hdl_t *hdl) 958{ 959 zfs_case_t *zcp; 960 uu_list_walk_t *walk; 961 libzfs_handle_t *zhdl; 962 963 /* 964 * Remove all active cases. 965 */ 966 walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST); 967 while ((zcp = uu_list_walk_next(walk)) != NULL) { 968 fmd_hdl_debug(hdl, "removing case ena %llu", 969 (long long unsigned)zcp->zc_data.zc_ena); 970 uu_list_remove(zfs_cases, zcp); 971 uu_list_node_fini(zcp, &zcp->zc_node, zfs_case_pool); 972 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t)); 973 } 974 uu_list_walk_end(walk); 975 976 uu_list_destroy(zfs_cases); 977 uu_list_pool_destroy(zfs_case_pool); 978 979 zhdl = fmd_hdl_getspecific(hdl); 980 libzfs_fini(zhdl); 981} 982