1168404Spjd/* 2168404Spjd * CDDL HEADER START 3168404Spjd * 4168404Spjd * The contents of this file are subject to the terms of the 5168404Spjd * Common Development and Distribution License (the "License"). 6168404Spjd * You may not use this file except in compliance with the License. 7168404Spjd * 8168404Spjd * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9168404Spjd * or http://www.opensolaris.org/os/licensing. 10168404Spjd * See the License for the specific language governing permissions 11168404Spjd * and limitations under the License. 12168404Spjd * 13168404Spjd * When distributing Covered Code, include this CDDL HEADER in each 14168404Spjd * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15168404Spjd * If applicable, add the following below this CDDL HEADER, with the 16168404Spjd * fields enclosed by brackets "[]" replaced with your own identifying 17168404Spjd * information: Portions Copyright [yyyy] [name of copyright owner] 18168404Spjd * 19168404Spjd * CDDL HEADER END 20168404Spjd */ 21168404Spjd/* 22219089Spjd * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23168404Spjd * Use is subject to license terms. 24168404Spjd */ 25240415Smm/* 26240415Smm * Copyright (c) 2012 by Delphix. All rights reserved. 27240415Smm */ 28168404Spjd 29168404Spjd/* 30168404Spjd * This file contains the code to implement file range locking in 31251631Sdelphij * ZFS, although there isn't much specific to ZFS (all that comes to mind is 32168404Spjd * support for growing the blocksize). 33168404Spjd * 34168404Spjd * Interface 35168404Spjd * --------- 36168404Spjd * Defined in zfs_rlock.h but essentially: 37168404Spjd * rl = zfs_range_lock(zp, off, len, lock_type); 38168404Spjd * zfs_range_unlock(rl); 39168404Spjd * zfs_range_reduce(rl, off, len); 40168404Spjd * 41168404Spjd * AVL tree 42168404Spjd * -------- 43168404Spjd * An AVL tree is used to maintain the state of the existing ranges 44168404Spjd * that are locked for exclusive (writer) or shared (reader) use. 45168404Spjd * The starting range offset is used for searching and sorting the tree. 46168404Spjd * 47168404Spjd * Common case 48168404Spjd * ----------- 49168404Spjd * The (hopefully) usual case is of no overlaps or contention for 50168404Spjd * locks. On entry to zfs_lock_range() a rl_t is allocated; the tree 51168404Spjd * searched that finds no overlap, and *this* rl_t is placed in the tree. 52168404Spjd * 53168404Spjd * Overlaps/Reference counting/Proxy locks 54168404Spjd * --------------------------------------- 55168404Spjd * The avl code only allows one node at a particular offset. Also it's very 56168404Spjd * inefficient to search through all previous entries looking for overlaps 57168404Spjd * (because the very 1st in the ordered list might be at offset 0 but 58168404Spjd * cover the whole file). 59168404Spjd * So this implementation uses reference counts and proxy range locks. 60168404Spjd * Firstly, only reader locks use reference counts and proxy locks, 61168404Spjd * because writer locks are exclusive. 62168404Spjd * When a reader lock overlaps with another then a proxy lock is created 63168404Spjd * for that range and replaces the original lock. If the overlap 64168404Spjd * is exact then the reference count of the proxy is simply incremented. 65168404Spjd * Otherwise, the proxy lock is split into smaller lock ranges and 66168404Spjd * new proxy locks created for non overlapping ranges. 67168404Spjd * The reference counts are adjusted accordingly. 68168404Spjd * Meanwhile, the orginal lock is kept around (this is the callers handle) 69168404Spjd * and its offset and length are used when releasing the lock. 70168404Spjd * 71168404Spjd * Thread coordination 72168404Spjd * ------------------- 73168404Spjd * In order to make wakeups efficient and to ensure multiple continuous 74168404Spjd * readers on a range don't starve a writer for the same range lock, 75168404Spjd * two condition variables are allocated in each rl_t. 76168404Spjd * If a writer (or reader) can't get a range it initialises the writer 77168404Spjd * (or reader) cv; sets a flag saying there's a writer (or reader) waiting; 78168404Spjd * and waits on that cv. When a thread unlocks that range it wakes up all 79168404Spjd * writers then all readers before destroying the lock. 80168404Spjd * 81168404Spjd * Append mode writes 82168404Spjd * ------------------ 83168404Spjd * Append mode writes need to lock a range at the end of a file. 84168404Spjd * The offset of the end of the file is determined under the 85168404Spjd * range locking mutex, and the lock type converted from RL_APPEND to 86168404Spjd * RL_WRITER and the range locked. 87168404Spjd * 88168404Spjd * Grow block handling 89168404Spjd * ------------------- 90168404Spjd * ZFS supports multiple block sizes currently upto 128K. The smallest 91168404Spjd * block size is used for the file which is grown as needed. During this 92168404Spjd * growth all other writers and readers must be excluded. 93168404Spjd * So if the block size needs to be grown then the whole file is 94168404Spjd * exclusively locked, then later the caller will reduce the lock 95168404Spjd * range to just the range to be written using zfs_reduce_range. 96168404Spjd */ 97168404Spjd 98168404Spjd#include <sys/zfs_rlock.h> 99168404Spjd 100168404Spjd/* 101168404Spjd * Check if a write lock can be grabbed, or wait and recheck until available. 102168404Spjd */ 103168404Spjdstatic void 104168404Spjdzfs_range_lock_writer(znode_t *zp, rl_t *new) 105168404Spjd{ 106168404Spjd avl_tree_t *tree = &zp->z_range_avl; 107168404Spjd rl_t *rl; 108168404Spjd avl_index_t where; 109168404Spjd uint64_t end_size; 110168404Spjd uint64_t off = new->r_off; 111168404Spjd uint64_t len = new->r_len; 112168404Spjd 113168404Spjd for (;;) { 114168404Spjd /* 115168404Spjd * Range locking is also used by zvol and uses a 116168404Spjd * dummied up znode. However, for zvol, we don't need to 117168404Spjd * append or grow blocksize, and besides we don't have 118219089Spjd * a "sa" data or z_zfsvfs - so skip that processing. 119168404Spjd * 120168404Spjd * Yes, this is ugly, and would be solved by not handling 121168404Spjd * grow or append in range lock code. If that was done then 122168404Spjd * we could make the range locking code generically available 123168404Spjd * to other non-zfs consumers. 124168404Spjd */ 125168404Spjd if (zp->z_vnode) { /* caller is ZPL */ 126168404Spjd /* 127168404Spjd * If in append mode pick up the current end of file. 128168404Spjd * This is done under z_range_lock to avoid races. 129168404Spjd */ 130168404Spjd if (new->r_type == RL_APPEND) 131219089Spjd new->r_off = zp->z_size; 132168404Spjd 133168404Spjd /* 134168404Spjd * If we need to grow the block size then grab the whole 135168404Spjd * file range. This is also done under z_range_lock to 136168404Spjd * avoid races. 137168404Spjd */ 138219089Spjd end_size = MAX(zp->z_size, new->r_off + len); 139168404Spjd if (end_size > zp->z_blksz && (!ISP2(zp->z_blksz) || 140168404Spjd zp->z_blksz < zp->z_zfsvfs->z_max_blksz)) { 141168404Spjd new->r_off = 0; 142168404Spjd new->r_len = UINT64_MAX; 143168404Spjd } 144168404Spjd } 145168404Spjd 146168404Spjd /* 147168404Spjd * First check for the usual case of no locks 148168404Spjd */ 149168404Spjd if (avl_numnodes(tree) == 0) { 150168404Spjd new->r_type = RL_WRITER; /* convert to writer */ 151168404Spjd avl_add(tree, new); 152168404Spjd return; 153168404Spjd } 154168404Spjd 155168404Spjd /* 156168404Spjd * Look for any locks in the range. 157168404Spjd */ 158168404Spjd rl = avl_find(tree, new, &where); 159168404Spjd if (rl) 160168404Spjd goto wait; /* already locked at same offset */ 161168404Spjd 162168404Spjd rl = (rl_t *)avl_nearest(tree, where, AVL_AFTER); 163168404Spjd if (rl && (rl->r_off < new->r_off + new->r_len)) 164168404Spjd goto wait; 165168404Spjd 166168404Spjd rl = (rl_t *)avl_nearest(tree, where, AVL_BEFORE); 167168404Spjd if (rl && rl->r_off + rl->r_len > new->r_off) 168168404Spjd goto wait; 169168404Spjd 170168404Spjd new->r_type = RL_WRITER; /* convert possible RL_APPEND */ 171168404Spjd avl_insert(tree, new, where); 172168404Spjd return; 173168404Spjdwait: 174168404Spjd if (!rl->r_write_wanted) { 175168404Spjd cv_init(&rl->r_wr_cv, NULL, CV_DEFAULT, NULL); 176168404Spjd rl->r_write_wanted = B_TRUE; 177168404Spjd } 178168404Spjd cv_wait(&rl->r_wr_cv, &zp->z_range_lock); 179168404Spjd 180168404Spjd /* reset to original */ 181168404Spjd new->r_off = off; 182168404Spjd new->r_len = len; 183168404Spjd } 184168404Spjd} 185168404Spjd 186168404Spjd/* 187168404Spjd * If this is an original (non-proxy) lock then replace it by 188168404Spjd * a proxy and return the proxy. 189168404Spjd */ 190168404Spjdstatic rl_t * 191168404Spjdzfs_range_proxify(avl_tree_t *tree, rl_t *rl) 192168404Spjd{ 193168404Spjd rl_t *proxy; 194168404Spjd 195168404Spjd if (rl->r_proxy) 196168404Spjd return (rl); /* already a proxy */ 197168404Spjd 198168404Spjd ASSERT3U(rl->r_cnt, ==, 1); 199168404Spjd ASSERT(rl->r_write_wanted == B_FALSE); 200168404Spjd ASSERT(rl->r_read_wanted == B_FALSE); 201168404Spjd avl_remove(tree, rl); 202168404Spjd rl->r_cnt = 0; 203168404Spjd 204168404Spjd /* create a proxy range lock */ 205168404Spjd proxy = kmem_alloc(sizeof (rl_t), KM_SLEEP); 206168404Spjd proxy->r_off = rl->r_off; 207168404Spjd proxy->r_len = rl->r_len; 208168404Spjd proxy->r_cnt = 1; 209168404Spjd proxy->r_type = RL_READER; 210168404Spjd proxy->r_proxy = B_TRUE; 211168404Spjd proxy->r_write_wanted = B_FALSE; 212168404Spjd proxy->r_read_wanted = B_FALSE; 213168404Spjd avl_add(tree, proxy); 214168404Spjd 215168404Spjd return (proxy); 216168404Spjd} 217168404Spjd 218168404Spjd/* 219168404Spjd * Split the range lock at the supplied offset 220168404Spjd * returning the *front* proxy. 221168404Spjd */ 222168404Spjdstatic rl_t * 223168404Spjdzfs_range_split(avl_tree_t *tree, rl_t *rl, uint64_t off) 224168404Spjd{ 225168404Spjd rl_t *front, *rear; 226168404Spjd 227168404Spjd ASSERT3U(rl->r_len, >, 1); 228168404Spjd ASSERT3U(off, >, rl->r_off); 229168404Spjd ASSERT3U(off, <, rl->r_off + rl->r_len); 230168404Spjd ASSERT(rl->r_write_wanted == B_FALSE); 231168404Spjd ASSERT(rl->r_read_wanted == B_FALSE); 232168404Spjd 233168404Spjd /* create the rear proxy range lock */ 234168404Spjd rear = kmem_alloc(sizeof (rl_t), KM_SLEEP); 235168404Spjd rear->r_off = off; 236168404Spjd rear->r_len = rl->r_off + rl->r_len - off; 237168404Spjd rear->r_cnt = rl->r_cnt; 238168404Spjd rear->r_type = RL_READER; 239168404Spjd rear->r_proxy = B_TRUE; 240168404Spjd rear->r_write_wanted = B_FALSE; 241168404Spjd rear->r_read_wanted = B_FALSE; 242168404Spjd 243168404Spjd front = zfs_range_proxify(tree, rl); 244168404Spjd front->r_len = off - rl->r_off; 245168404Spjd 246168404Spjd avl_insert_here(tree, rear, front, AVL_AFTER); 247168404Spjd return (front); 248168404Spjd} 249168404Spjd 250168404Spjd/* 251168404Spjd * Create and add a new proxy range lock for the supplied range. 252168404Spjd */ 253168404Spjdstatic void 254168404Spjdzfs_range_new_proxy(avl_tree_t *tree, uint64_t off, uint64_t len) 255168404Spjd{ 256168404Spjd rl_t *rl; 257168404Spjd 258168404Spjd ASSERT(len); 259168404Spjd rl = kmem_alloc(sizeof (rl_t), KM_SLEEP); 260168404Spjd rl->r_off = off; 261168404Spjd rl->r_len = len; 262168404Spjd rl->r_cnt = 1; 263168404Spjd rl->r_type = RL_READER; 264168404Spjd rl->r_proxy = B_TRUE; 265168404Spjd rl->r_write_wanted = B_FALSE; 266168404Spjd rl->r_read_wanted = B_FALSE; 267168404Spjd avl_add(tree, rl); 268168404Spjd} 269168404Spjd 270168404Spjdstatic void 271168404Spjdzfs_range_add_reader(avl_tree_t *tree, rl_t *new, rl_t *prev, avl_index_t where) 272168404Spjd{ 273168404Spjd rl_t *next; 274168404Spjd uint64_t off = new->r_off; 275168404Spjd uint64_t len = new->r_len; 276168404Spjd 277168404Spjd /* 278168404Spjd * prev arrives either: 279168404Spjd * - pointing to an entry at the same offset 280168404Spjd * - pointing to the entry with the closest previous offset whose 281168404Spjd * range may overlap with the new range 282168404Spjd * - null, if there were no ranges starting before the new one 283168404Spjd */ 284168404Spjd if (prev) { 285168404Spjd if (prev->r_off + prev->r_len <= off) { 286168404Spjd prev = NULL; 287168404Spjd } else if (prev->r_off != off) { 288168404Spjd /* 289168404Spjd * convert to proxy if needed then 290168404Spjd * split this entry and bump ref count 291168404Spjd */ 292168404Spjd prev = zfs_range_split(tree, prev, off); 293168404Spjd prev = AVL_NEXT(tree, prev); /* move to rear range */ 294168404Spjd } 295168404Spjd } 296168404Spjd ASSERT((prev == NULL) || (prev->r_off == off)); 297168404Spjd 298168404Spjd if (prev) 299168404Spjd next = prev; 300168404Spjd else 301168404Spjd next = (rl_t *)avl_nearest(tree, where, AVL_AFTER); 302168404Spjd 303168404Spjd if (next == NULL || off + len <= next->r_off) { 304168404Spjd /* no overlaps, use the original new rl_t in the tree */ 305168404Spjd avl_insert(tree, new, where); 306168404Spjd return; 307168404Spjd } 308168404Spjd 309168404Spjd if (off < next->r_off) { 310168404Spjd /* Add a proxy for initial range before the overlap */ 311168404Spjd zfs_range_new_proxy(tree, off, next->r_off - off); 312168404Spjd } 313168404Spjd 314168404Spjd new->r_cnt = 0; /* will use proxies in tree */ 315168404Spjd /* 316168404Spjd * We now search forward through the ranges, until we go past the end 317168404Spjd * of the new range. For each entry we make it a proxy if it 318168404Spjd * isn't already, then bump its reference count. If there's any 319168404Spjd * gaps between the ranges then we create a new proxy range. 320168404Spjd */ 321168404Spjd for (prev = NULL; next; prev = next, next = AVL_NEXT(tree, next)) { 322168404Spjd if (off + len <= next->r_off) 323168404Spjd break; 324168404Spjd if (prev && prev->r_off + prev->r_len < next->r_off) { 325168404Spjd /* there's a gap */ 326168404Spjd ASSERT3U(next->r_off, >, prev->r_off + prev->r_len); 327168404Spjd zfs_range_new_proxy(tree, prev->r_off + prev->r_len, 328168404Spjd next->r_off - (prev->r_off + prev->r_len)); 329168404Spjd } 330168404Spjd if (off + len == next->r_off + next->r_len) { 331168404Spjd /* exact overlap with end */ 332168404Spjd next = zfs_range_proxify(tree, next); 333168404Spjd next->r_cnt++; 334168404Spjd return; 335168404Spjd } 336168404Spjd if (off + len < next->r_off + next->r_len) { 337168404Spjd /* new range ends in the middle of this block */ 338168404Spjd next = zfs_range_split(tree, next, off + len); 339168404Spjd next->r_cnt++; 340168404Spjd return; 341168404Spjd } 342168404Spjd ASSERT3U(off + len, >, next->r_off + next->r_len); 343168404Spjd next = zfs_range_proxify(tree, next); 344168404Spjd next->r_cnt++; 345168404Spjd } 346168404Spjd 347168404Spjd /* Add the remaining end range. */ 348168404Spjd zfs_range_new_proxy(tree, prev->r_off + prev->r_len, 349168404Spjd (off + len) - (prev->r_off + prev->r_len)); 350168404Spjd} 351168404Spjd 352168404Spjd/* 353168404Spjd * Check if a reader lock can be grabbed, or wait and recheck until available. 354168404Spjd */ 355168404Spjdstatic void 356168404Spjdzfs_range_lock_reader(znode_t *zp, rl_t *new) 357168404Spjd{ 358168404Spjd avl_tree_t *tree = &zp->z_range_avl; 359168404Spjd rl_t *prev, *next; 360168404Spjd avl_index_t where; 361168404Spjd uint64_t off = new->r_off; 362168404Spjd uint64_t len = new->r_len; 363168404Spjd 364168404Spjd /* 365168404Spjd * Look for any writer locks in the range. 366168404Spjd */ 367168404Spjdretry: 368168404Spjd prev = avl_find(tree, new, &where); 369168404Spjd if (prev == NULL) 370168404Spjd prev = (rl_t *)avl_nearest(tree, where, AVL_BEFORE); 371168404Spjd 372168404Spjd /* 373168404Spjd * Check the previous range for a writer lock overlap. 374168404Spjd */ 375168404Spjd if (prev && (off < prev->r_off + prev->r_len)) { 376168404Spjd if ((prev->r_type == RL_WRITER) || (prev->r_write_wanted)) { 377168404Spjd if (!prev->r_read_wanted) { 378168404Spjd cv_init(&prev->r_rd_cv, NULL, CV_DEFAULT, NULL); 379168404Spjd prev->r_read_wanted = B_TRUE; 380168404Spjd } 381168404Spjd cv_wait(&prev->r_rd_cv, &zp->z_range_lock); 382168404Spjd goto retry; 383168404Spjd } 384168404Spjd if (off + len < prev->r_off + prev->r_len) 385168404Spjd goto got_lock; 386168404Spjd } 387168404Spjd 388168404Spjd /* 389168404Spjd * Search through the following ranges to see if there's 390168404Spjd * write lock any overlap. 391168404Spjd */ 392168404Spjd if (prev) 393168404Spjd next = AVL_NEXT(tree, prev); 394168404Spjd else 395168404Spjd next = (rl_t *)avl_nearest(tree, where, AVL_AFTER); 396168404Spjd for (; next; next = AVL_NEXT(tree, next)) { 397168404Spjd if (off + len <= next->r_off) 398168404Spjd goto got_lock; 399168404Spjd if ((next->r_type == RL_WRITER) || (next->r_write_wanted)) { 400168404Spjd if (!next->r_read_wanted) { 401168404Spjd cv_init(&next->r_rd_cv, NULL, CV_DEFAULT, NULL); 402168404Spjd next->r_read_wanted = B_TRUE; 403168404Spjd } 404168404Spjd cv_wait(&next->r_rd_cv, &zp->z_range_lock); 405168404Spjd goto retry; 406168404Spjd } 407168404Spjd if (off + len <= next->r_off + next->r_len) 408168404Spjd goto got_lock; 409168404Spjd } 410168404Spjd 411168404Spjdgot_lock: 412168404Spjd /* 413168404Spjd * Add the read lock, which may involve splitting existing 414168404Spjd * locks and bumping ref counts (r_cnt). 415168404Spjd */ 416168404Spjd zfs_range_add_reader(tree, new, prev, where); 417168404Spjd} 418168404Spjd 419168404Spjd/* 420168404Spjd * Lock a range (offset, length) as either shared (RL_READER) 421168404Spjd * or exclusive (RL_WRITER). Returns the range lock structure 422168404Spjd * for later unlocking or reduce range (if entire file 423168404Spjd * previously locked as RL_WRITER). 424168404Spjd */ 425168404Spjdrl_t * 426168404Spjdzfs_range_lock(znode_t *zp, uint64_t off, uint64_t len, rl_type_t type) 427168404Spjd{ 428168404Spjd rl_t *new; 429168404Spjd 430168404Spjd ASSERT(type == RL_READER || type == RL_WRITER || type == RL_APPEND); 431168404Spjd 432168404Spjd new = kmem_alloc(sizeof (rl_t), KM_SLEEP); 433168404Spjd new->r_zp = zp; 434168404Spjd new->r_off = off; 435209962Smm if (len + off < off) /* overflow */ 436209962Smm len = UINT64_MAX - off; 437168404Spjd new->r_len = len; 438168404Spjd new->r_cnt = 1; /* assume it's going to be in the tree */ 439168404Spjd new->r_type = type; 440168404Spjd new->r_proxy = B_FALSE; 441168404Spjd new->r_write_wanted = B_FALSE; 442168404Spjd new->r_read_wanted = B_FALSE; 443168404Spjd 444168404Spjd mutex_enter(&zp->z_range_lock); 445168404Spjd if (type == RL_READER) { 446168404Spjd /* 447168404Spjd * First check for the usual case of no locks 448168404Spjd */ 449168404Spjd if (avl_numnodes(&zp->z_range_avl) == 0) 450168404Spjd avl_add(&zp->z_range_avl, new); 451168404Spjd else 452168404Spjd zfs_range_lock_reader(zp, new); 453168404Spjd } else 454168404Spjd zfs_range_lock_writer(zp, new); /* RL_WRITER or RL_APPEND */ 455168404Spjd mutex_exit(&zp->z_range_lock); 456168404Spjd return (new); 457168404Spjd} 458168404Spjd 459168404Spjd/* 460168404Spjd * Unlock a reader lock 461168404Spjd */ 462168404Spjdstatic void 463168404Spjdzfs_range_unlock_reader(znode_t *zp, rl_t *remove) 464168404Spjd{ 465168404Spjd avl_tree_t *tree = &zp->z_range_avl; 466247187Smm rl_t *rl, *next = NULL; 467168404Spjd uint64_t len; 468168404Spjd 469168404Spjd /* 470168404Spjd * The common case is when the remove entry is in the tree 471168404Spjd * (cnt == 1) meaning there's been no other reader locks overlapping 472168404Spjd * with this one. Otherwise the remove entry will have been 473168404Spjd * removed from the tree and replaced by proxies (one or 474168404Spjd * more ranges mapping to the entire range). 475168404Spjd */ 476168404Spjd if (remove->r_cnt == 1) { 477168404Spjd avl_remove(tree, remove); 478185029Spjd if (remove->r_write_wanted) { 479168404Spjd cv_broadcast(&remove->r_wr_cv); 480185029Spjd cv_destroy(&remove->r_wr_cv); 481185029Spjd } 482185029Spjd if (remove->r_read_wanted) { 483168404Spjd cv_broadcast(&remove->r_rd_cv); 484185029Spjd cv_destroy(&remove->r_rd_cv); 485185029Spjd } 486168404Spjd } else { 487240415Smm ASSERT0(remove->r_cnt); 488240415Smm ASSERT0(remove->r_write_wanted); 489240415Smm ASSERT0(remove->r_read_wanted); 490168404Spjd /* 491168404Spjd * Find start proxy representing this reader lock, 492168404Spjd * then decrement ref count on all proxies 493168404Spjd * that make up this range, freeing them as needed. 494168404Spjd */ 495168404Spjd rl = avl_find(tree, remove, NULL); 496168404Spjd ASSERT(rl); 497168404Spjd ASSERT(rl->r_cnt); 498168404Spjd ASSERT(rl->r_type == RL_READER); 499168404Spjd for (len = remove->r_len; len != 0; rl = next) { 500168404Spjd len -= rl->r_len; 501168404Spjd if (len) { 502168404Spjd next = AVL_NEXT(tree, rl); 503168404Spjd ASSERT(next); 504168404Spjd ASSERT(rl->r_off + rl->r_len == next->r_off); 505168404Spjd ASSERT(next->r_cnt); 506168404Spjd ASSERT(next->r_type == RL_READER); 507168404Spjd } 508168404Spjd rl->r_cnt--; 509168404Spjd if (rl->r_cnt == 0) { 510168404Spjd avl_remove(tree, rl); 511185029Spjd if (rl->r_write_wanted) { 512168404Spjd cv_broadcast(&rl->r_wr_cv); 513185029Spjd cv_destroy(&rl->r_wr_cv); 514185029Spjd } 515185029Spjd if (rl->r_read_wanted) { 516168404Spjd cv_broadcast(&rl->r_rd_cv); 517185029Spjd cv_destroy(&rl->r_rd_cv); 518185029Spjd } 519168404Spjd kmem_free(rl, sizeof (rl_t)); 520168404Spjd } 521168404Spjd } 522168404Spjd } 523168404Spjd kmem_free(remove, sizeof (rl_t)); 524168404Spjd} 525168404Spjd 526168404Spjd/* 527168404Spjd * Unlock range and destroy range lock structure. 528168404Spjd */ 529168404Spjdvoid 530168404Spjdzfs_range_unlock(rl_t *rl) 531168404Spjd{ 532168404Spjd znode_t *zp = rl->r_zp; 533168404Spjd 534168404Spjd ASSERT(rl->r_type == RL_WRITER || rl->r_type == RL_READER); 535168404Spjd ASSERT(rl->r_cnt == 1 || rl->r_cnt == 0); 536168404Spjd ASSERT(!rl->r_proxy); 537168404Spjd 538168404Spjd mutex_enter(&zp->z_range_lock); 539168404Spjd if (rl->r_type == RL_WRITER) { 540168404Spjd /* writer locks can't be shared or split */ 541168404Spjd avl_remove(&zp->z_range_avl, rl); 542168404Spjd mutex_exit(&zp->z_range_lock); 543168404Spjd if (rl->r_write_wanted) { 544168404Spjd cv_broadcast(&rl->r_wr_cv); 545168404Spjd cv_destroy(&rl->r_wr_cv); 546168404Spjd } 547168404Spjd if (rl->r_read_wanted) { 548168404Spjd cv_broadcast(&rl->r_rd_cv); 549168404Spjd cv_destroy(&rl->r_rd_cv); 550168404Spjd } 551168404Spjd kmem_free(rl, sizeof (rl_t)); 552168404Spjd } else { 553168404Spjd /* 554168404Spjd * lock may be shared, let zfs_range_unlock_reader() 555168404Spjd * release the lock and free the rl_t 556168404Spjd */ 557168404Spjd zfs_range_unlock_reader(zp, rl); 558168404Spjd mutex_exit(&zp->z_range_lock); 559168404Spjd } 560168404Spjd} 561168404Spjd 562168404Spjd/* 563168404Spjd * Reduce range locked as RL_WRITER from whole file to specified range. 564168404Spjd * Asserts the whole file is exclusivly locked and so there's only one 565168404Spjd * entry in the tree. 566168404Spjd */ 567168404Spjdvoid 568168404Spjdzfs_range_reduce(rl_t *rl, uint64_t off, uint64_t len) 569168404Spjd{ 570168404Spjd znode_t *zp = rl->r_zp; 571168404Spjd 572168404Spjd /* Ensure there are no other locks */ 573168404Spjd ASSERT(avl_numnodes(&zp->z_range_avl) == 1); 574168404Spjd ASSERT(rl->r_off == 0); 575168404Spjd ASSERT(rl->r_type == RL_WRITER); 576168404Spjd ASSERT(!rl->r_proxy); 577168404Spjd ASSERT3U(rl->r_len, ==, UINT64_MAX); 578168404Spjd ASSERT3U(rl->r_cnt, ==, 1); 579168404Spjd 580168404Spjd mutex_enter(&zp->z_range_lock); 581168404Spjd rl->r_off = off; 582168404Spjd rl->r_len = len; 583168404Spjd mutex_exit(&zp->z_range_lock); 584168404Spjd if (rl->r_write_wanted) 585168404Spjd cv_broadcast(&rl->r_wr_cv); 586168404Spjd if (rl->r_read_wanted) 587168404Spjd cv_broadcast(&rl->r_rd_cv); 588168404Spjd} 589168404Spjd 590168404Spjd/* 591168404Spjd * AVL comparison function used to order range locks 592168404Spjd * Locks are ordered on the start offset of the range. 593168404Spjd */ 594168404Spjdint 595168404Spjdzfs_range_compare(const void *arg1, const void *arg2) 596168404Spjd{ 597168404Spjd const rl_t *rl1 = arg1; 598168404Spjd const rl_t *rl2 = arg2; 599168404Spjd 600168404Spjd if (rl1->r_off > rl2->r_off) 601168404Spjd return (1); 602168404Spjd if (rl1->r_off < rl2->r_off) 603168404Spjd return (-1); 604168404Spjd return (0); 605168404Spjd} 606