1219089Spjd/* 2219089Spjd * CDDL HEADER START 3219089Spjd * 4219089Spjd * The contents of this file are subject to the terms of the 5219089Spjd * Common Development and Distribution License (the "License"). 6219089Spjd * You may not use this file except in compliance with the License. 7219089Spjd * 8219089Spjd * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9219089Spjd * or http://www.opensolaris.org/os/licensing. 10219089Spjd * See the License for the specific language governing permissions 11219089Spjd * and limitations under the License. 12219089Spjd * 13219089Spjd * When distributing Covered Code, include this CDDL HEADER in each 14219089Spjd * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15219089Spjd * If applicable, add the following below this CDDL HEADER, with the 16219089Spjd * fields enclosed by brackets "[]" replaced with your own identifying 17219089Spjd * information: Portions Copyright [yyyy] [name of copyright owner] 18219089Spjd * 19219089Spjd * CDDL HEADER END 20219089Spjd */ 21219089Spjd/* 22219089Spjd * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 23289562Smav * Copyright (c) 2014, 2015 by Delphix. All rights reserved. 24219089Spjd */ 25219089Spjd 26219089Spjd/* 27219089Spjd * A Zero Reference Lock (ZRL) is a reference count that can lock out new 28219089Spjd * references only when the count is zero and only without waiting if the count 29219089Spjd * is not already zero. It is similar to a read-write lock in that it allows 30219089Spjd * multiple readers and only a single writer, but it does not allow a writer to 31219089Spjd * block while waiting for readers to exit, and therefore the question of 32219089Spjd * reader/writer priority is moot (no WRWANT bit). Since the equivalent of 33219089Spjd * rw_enter(&lock, RW_WRITER) is disallowed and only tryenter() is allowed, it 34219089Spjd * is perfectly safe for the same reader to acquire the same lock multiple 35219089Spjd * times. The fact that a ZRL is reentrant for readers (through multiple calls 36219089Spjd * to zrl_add()) makes it convenient for determining whether something is 37219089Spjd * actively referenced without the fuss of flagging lock ownership across 38219089Spjd * function calls. 39219089Spjd */ 40219089Spjd#include <sys/zrlock.h> 41219089Spjd 42219089Spjd/* 43219089Spjd * A ZRL can be locked only while there are zero references, so ZRL_LOCKED is 44219089Spjd * treated as zero references. 45219089Spjd */ 46286605Smav#define ZRL_LOCKED -1 47219089Spjd#define ZRL_DESTROYED -2 48219089Spjd 49219089Spjdvoid 50219089Spjdzrl_init(zrlock_t *zrl) 51219089Spjd{ 52219089Spjd mutex_init(&zrl->zr_mtx, NULL, MUTEX_DEFAULT, NULL); 53219089Spjd zrl->zr_refcount = 0; 54219089Spjd cv_init(&zrl->zr_cv, NULL, CV_DEFAULT, NULL); 55219089Spjd#ifdef ZFS_DEBUG 56219089Spjd zrl->zr_owner = NULL; 57219089Spjd zrl->zr_caller = NULL; 58219089Spjd#endif 59219089Spjd} 60219089Spjd 61219089Spjdvoid 62219089Spjdzrl_destroy(zrlock_t *zrl) 63219089Spjd{ 64286605Smav ASSERT0(zrl->zr_refcount); 65219089Spjd 66219089Spjd mutex_destroy(&zrl->zr_mtx); 67219089Spjd zrl->zr_refcount = ZRL_DESTROYED; 68219089Spjd cv_destroy(&zrl->zr_cv); 69219089Spjd} 70219089Spjd 71219089Spjdvoid 72289562Smavzrl_add_impl(zrlock_t *zrl, const char *zc) 73219089Spjd{ 74219089Spjd uint32_t n = (uint32_t)zrl->zr_refcount; 75219089Spjd 76219089Spjd while (n != ZRL_LOCKED) { 77219089Spjd uint32_t cas = atomic_cas_32( 78219089Spjd (uint32_t *)&zrl->zr_refcount, n, n + 1); 79219089Spjd if (cas == n) { 80286605Smav ASSERT3S((int32_t)n, >=, 0); 81219089Spjd#ifdef ZFS_DEBUG 82219089Spjd if (zrl->zr_owner == curthread) { 83219089Spjd DTRACE_PROBE2(zrlock__reentry, 84219089Spjd zrlock_t *, zrl, uint32_t, n); 85219089Spjd } 86219089Spjd zrl->zr_owner = curthread; 87219089Spjd zrl->zr_caller = zc; 88219089Spjd#endif 89219089Spjd return; 90219089Spjd } 91219089Spjd n = cas; 92219089Spjd } 93219089Spjd 94219089Spjd mutex_enter(&zrl->zr_mtx); 95219089Spjd while (zrl->zr_refcount == ZRL_LOCKED) { 96219089Spjd cv_wait(&zrl->zr_cv, &zrl->zr_mtx); 97219089Spjd } 98286605Smav ASSERT3S(zrl->zr_refcount, >=, 0); 99219089Spjd zrl->zr_refcount++; 100219089Spjd#ifdef ZFS_DEBUG 101219089Spjd zrl->zr_owner = curthread; 102219089Spjd zrl->zr_caller = zc; 103219089Spjd#endif 104219089Spjd mutex_exit(&zrl->zr_mtx); 105219089Spjd} 106219089Spjd 107219089Spjdvoid 108219089Spjdzrl_remove(zrlock_t *zrl) 109219089Spjd{ 110219089Spjd uint32_t n; 111219089Spjd 112219089Spjd#ifdef ZFS_DEBUG 113219089Spjd if (zrl->zr_owner == curthread) { 114219089Spjd zrl->zr_owner = NULL; 115219089Spjd zrl->zr_caller = NULL; 116219089Spjd } 117219089Spjd#endif 118286605Smav n = atomic_dec_32_nv((uint32_t *)&zrl->zr_refcount); 119286605Smav ASSERT3S((int32_t)n, >=, 0); 120219089Spjd} 121219089Spjd 122219089Spjdint 123219089Spjdzrl_tryenter(zrlock_t *zrl) 124219089Spjd{ 125219089Spjd uint32_t n = (uint32_t)zrl->zr_refcount; 126219089Spjd 127219089Spjd if (n == 0) { 128219089Spjd uint32_t cas = atomic_cas_32( 129219089Spjd (uint32_t *)&zrl->zr_refcount, 0, ZRL_LOCKED); 130219089Spjd if (cas == 0) { 131219089Spjd#ifdef ZFS_DEBUG 132286605Smav ASSERT3P(zrl->zr_owner, ==, NULL); 133219089Spjd zrl->zr_owner = curthread; 134219089Spjd#endif 135219089Spjd return (1); 136219089Spjd } 137219089Spjd } 138219089Spjd 139286605Smav ASSERT3S((int32_t)n, >, ZRL_DESTROYED); 140219089Spjd 141219089Spjd return (0); 142219089Spjd} 143219089Spjd 144219089Spjdvoid 145219089Spjdzrl_exit(zrlock_t *zrl) 146219089Spjd{ 147286605Smav ASSERT3S(zrl->zr_refcount, ==, ZRL_LOCKED); 148219089Spjd 149219089Spjd mutex_enter(&zrl->zr_mtx); 150219089Spjd#ifdef ZFS_DEBUG 151286605Smav ASSERT3P(zrl->zr_owner, ==, curthread); 152219089Spjd zrl->zr_owner = NULL; 153219089Spjd membar_producer(); /* make sure the owner store happens first */ 154219089Spjd#endif 155219089Spjd zrl->zr_refcount = 0; 156219089Spjd cv_broadcast(&zrl->zr_cv); 157219089Spjd mutex_exit(&zrl->zr_mtx); 158219089Spjd} 159219089Spjd 160219089Spjdint 161219089Spjdzrl_refcount(zrlock_t *zrl) 162219089Spjd{ 163286605Smav ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED); 164219089Spjd 165219089Spjd int n = (int)zrl->zr_refcount; 166219089Spjd return (n <= 0 ? 0 : n); 167219089Spjd} 168219089Spjd 169219089Spjdint 170219089Spjdzrl_is_zero(zrlock_t *zrl) 171219089Spjd{ 172286605Smav ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED); 173219089Spjd 174219089Spjd return (zrl->zr_refcount <= 0); 175219089Spjd} 176219089Spjd 177219089Spjdint 178219089Spjdzrl_is_locked(zrlock_t *zrl) 179219089Spjd{ 180286605Smav ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED); 181219089Spjd 182219089Spjd return (zrl->zr_refcount == ZRL_LOCKED); 183219089Spjd} 184219089Spjd 185219089Spjd#ifdef ZFS_DEBUG 186219089Spjdkthread_t * 187219089Spjdzrl_owner(zrlock_t *zrl) 188219089Spjd{ 189219089Spjd return (zrl->zr_owner); 190219089Spjd} 191219089Spjd#endif 192