sys/kern/vfs_cache.c

235 NCP2NEGLIST(struct namecache *ncp)
238 	return (&neglists[(((uintptr_t)(ncp) >> 8) & ncneghash)]);
275 	struct namecache *ncp;
282 		ncp = &ncp_ts->nc_nc;
285 			ncp = uma_zalloc(cache_zone_small, M_WAITOK);
287 			ncp = uma_zalloc(cache_zone_large, M_WAITOK);
289 	return (ncp);
293 cache_free(struct namecache *ncp)
297 	if (ncp == NULL)
299 	if ((ncp->nc_flag & NCF_DVDROP) != 0)
300 		vdrop(ncp->nc_dvp);
301 	if (__predict_false(ncp->nc_flag & NCF_TS)) {
302 		ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
303 		if (ncp->nc_nlen <= CACHE_PATH_CUTOFF)
308 		if (ncp->nc_nlen <= CACHE_PATH_CUTOFF)
309 			uma_zfree(cache_zone_small, ncp);
311 			uma_zfree(cache_zone_large, ncp);
316 cache_out_ts(struct namecache *ncp, struct timespec *tsp, int *ticksp)
320 	KASSERT((ncp->nc_flag & NCF_TS) != 0 ||
327 	ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
379 static void cache_zap_locked(struct namecache *ncp, bool neg_locked);
427 NCP2BUCKETLOCK(struct namecache *ncp)
431 	hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, ncp->nc_dvp);
437 cache_assert_bucket_locked(struct namecache *ncp, int mode)
441 	blp = NCP2BUCKETLOCK(ncp);
562 	struct namecache *ncp;
578 		LIST_FOREACH(ncp, ncpp, nc_hash)
596 	struct namecache *ncp;
611 		LIST_FOREACH(ncp, ncpp, nc_hash) {
653 cache_negative_hit(struct namecache *ncp)
658 	MPASS(ncp->nc_flag & NCF_NEGATIVE);
659 	hits = atomic_fetchadd_int(&ncp->nc_neghits, 1);
660 	if (ncp->nc_flag & NCF_HOTNEGATIVE) {
664 		if (ncp->nc_flag & NCF_HOTNEGATIVE) {
665 			TAILQ_REMOVE(&ncneg_hot.nl_list, ncp, nc_dst);
666 			TAILQ_INSERT_TAIL(&ncneg_hot.nl_list, ncp, nc_dst);
677 	neglist = NCP2NEGLIST(ncp);
679 	if (!(ncp->nc_flag & NCF_HOTNEGATIVE)) {
680 		TAILQ_REMOVE(&neglist->nl_list, ncp, nc_dst);
681 		TAILQ_INSERT_TAIL(&ncneg_hot.nl_list, ncp, nc_dst);
682 		ncp->nc_flag |= NCF_HOTNEGATIVE;
689 cache_negative_insert(struct namecache *ncp, bool neg_locked)
693 	MPASS(ncp->nc_flag & NCF_NEGATIVE);
694 	cache_assert_bucket_locked(ncp, RA_WLOCKED);
695 	neglist = NCP2NEGLIST(ncp);
701 	TAILQ_INSERT_TAIL(&neglist->nl_list, ncp, nc_dst);
708 cache_negative_remove(struct namecache *ncp, bool neg_locked)
714 	MPASS(ncp->nc_flag & NCF_NEGATIVE);
715 	cache_assert_bucket_locked(ncp, RA_WLOCKED);
716 	neglist = NCP2NEGLIST(ncp);
718 		if (ncp->nc_flag & NCF_HOTNEGATIVE) {
721 			if (!(ncp->nc_flag & NCF_HOTNEGATIVE)) {
730 	if (ncp->nc_flag & NCF_HOTNEGATIVE) {
732 		TAILQ_REMOVE(&ncneg_hot.nl_list, ncp, nc_dst);
735 		TAILQ_REMOVE(&neglist->nl_list, ncp, nc_dst);
749 	struct namecache *ncp;
752 	*ncpp = ncp = NULL;
759 		ncp = TAILQ_FIRST(&neglist->nl_list);
760 		if (ncp != NULL)
766 	*ncpp = ncp;
772 	struct namecache *ncp, *ncp2;
781 	ncp = TAILQ_FIRST(&ncneg_hot.nl_list);
782 	if (ncp != NULL) {
783 		neglist = NCP2NEGLIST(ncp);
785 		TAILQ_REMOVE(&ncneg_hot.nl_list, ncp, nc_dst);
786 		TAILQ_INSERT_TAIL(&neglist->nl_list, ncp, nc_dst);
787 		ncp->nc_flag &= ~NCF_HOTNEGATIVE;
791 	cache_negative_shrink_select(shrink_list_turn, &ncp, &neglist);
795 	if (ncp == NULL && shrink_list_turn == 0)
796 		cache_negative_shrink_select(shrink_list_turn, &ncp, &neglist);
797 	if (ncp == NULL) {
802 	MPASS(ncp->nc_flag & NCF_NEGATIVE);
803 	dvlp = VP2VNODELOCK(ncp->nc_dvp);
804 	blp = NCP2BUCKETLOCK(ncp);
811 	if (ncp != ncp2 || dvlp != VP2VNODELOCK(ncp2->nc_dvp) ||
813 		ncp = NULL;
816 	SDT_PROBE3(vfs, namecache, shrink_negative, done, ncp->nc_dvp,
817 	    ncp->nc_name, ncp->nc_neghits);
819 	cache_zap_locked(ncp, true);
826 	cache_free(ncp);
836 cache_zap_locked(struct namecache *ncp, bool neg_locked)
839 	if (!(ncp->nc_flag & NCF_NEGATIVE))
840 		cache_assert_vnode_locked(ncp->nc_vp);
841 	cache_assert_vnode_locked(ncp->nc_dvp);
842 	cache_assert_bucket_locked(ncp, RA_WLOCKED);
844 	CTR2(KTR_VFS, "cache_zap(%p) vp %p", ncp,
845 	    (ncp->nc_flag & NCF_NEGATIVE) ? NULL : ncp->nc_vp);
846 	if (!(ncp->nc_flag & NCF_NEGATIVE)) {
847 		SDT_PROBE3(vfs, namecache, zap, done, ncp->nc_dvp,
848 		    ncp->nc_name, ncp->nc_vp);
850 		SDT_PROBE3(vfs, namecache, zap_negative, done, ncp->nc_dvp,
851 		    ncp->nc_name, ncp->nc_neghits);
853 	LIST_REMOVE(ncp, nc_hash);
854 	if (!(ncp->nc_flag & NCF_NEGATIVE)) {
855 		TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst, ncp, nc_dst);
856 		if (ncp == ncp->nc_vp->v_cache_dd)
857 			ncp->nc_vp->v_cache_dd = NULL;
859 		cache_negative_remove(ncp, neg_locked);
861 	if (ncp->nc_flag & NCF_ISDOTDOT) {
862 		if (ncp == ncp->nc_dvp->v_cache_dd)
863 			ncp->nc_dvp->v_cache_dd = NULL;
865 		LIST_REMOVE(ncp, nc_src);
866 		if (LIST_EMPTY(&ncp->nc_dvp->v_cache_src)) {
867 			ncp->nc_flag |= NCF_DVDROP;
875 cache_zap_negative_locked_vnode_kl(struct namecache *ncp, struct vnode *vp)
879 	MPASS(ncp->nc_dvp == vp);
880 	MPASS(ncp->nc_flag & NCF_NEGATIVE);
883 	blp = NCP2BUCKETLOCK(ncp);
885 	cache_zap_locked(ncp, false);
890 cache_zap_locked_vnode_kl2(struct namecache *ncp, struct vnode *vp,
896 	MPASS(vp == ncp->nc_dvp || vp == ncp->nc_vp);
899 	if (ncp->nc_flag & NCF_NEGATIVE) {
904 		cache_zap_negative_locked_vnode_kl(ncp, vp);
909 	blp = NCP2BUCKETLOCK(ncp);
910 	vlp1 = VP2VNODELOCK(ncp->nc_dvp);
911 	vlp2 = VP2VNODELOCK(ncp->nc_vp);
932 	cache_zap_locked(ncp, false);
948 cache_zap_locked_vnode(struct namecache *ncp, struct vnode *vp)
954 	MPASS(vp == ncp->nc_dvp || vp == ncp->nc_vp);
958 	if (ncp->nc_flag & NCF_NEGATIVE) {
959 		cache_zap_negative_locked_vnode_kl(ncp, vp);
963 	blp = NCP2BUCKETLOCK(ncp);
964 	vlp1 = VP2VNODELOCK(ncp->nc_dvp);
965 	vlp2 = VP2VNODELOCK(ncp->nc_vp);
978 	cache_zap_locked(ncp, false);
987 cache_zap_wlocked_bucket(struct namecache *ncp, struct rwlock *blp)
991 	cache_assert_bucket_locked(ncp, RA_WLOCKED);
993 	dvlp = VP2VNODELOCK(ncp->nc_dvp);
995 	if (!(ncp->nc_flag & NCF_NEGATIVE))
996 		vlp = VP2VNODELOCK(ncp->nc_vp);
998 		cache_zap_locked(ncp, false);
1009 cache_zap_rlocked_bucket(struct namecache *ncp, struct rwlock *blp)
1013 	cache_assert_bucket_locked(ncp, RA_RLOCKED);
1015 	dvlp = VP2VNODELOCK(ncp->nc_dvp);
1017 	if (!(ncp->nc_flag & NCF_NEGATIVE))
1018 		vlp = VP2VNODELOCK(ncp->nc_vp);
1022 		cache_zap_locked(ncp, false);
1033 cache_zap_wlocked_bucket_kl(struct namecache *ncp, struct rwlock *blp,
1038 	cache_assert_bucket_locked(ncp, RA_WLOCKED);
1040 	dvlp = VP2VNODELOCK(ncp->nc_dvp);
1042 	if (!(ncp->nc_flag & NCF_NEGATIVE))
1043 		vlp = VP2VNODELOCK(ncp->nc_vp);
1047 		cache_zap_locked(ncp, false);
1062 		cache_zap_locked(ncp, false);
1128 	struct namecache *ncp;
1141 		ncp = dvp->v_cache_dd;
1142 		if (ncp == NULL) {
1150 		if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) {
1151 			if (ncp->nc_dvp != dvp)
1152 				panic("dvp %p v_cache_dd %p\n", dvp, ncp);
1153 			if (!cache_zap_locked_vnode_kl2(ncp,
1160 			cache_free(ncp);
1178 	LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
1180 		if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
1181 		    !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
1186 	if (ncp == NULL) {
1193 	error = cache_zap_wlocked_bucket(ncp, blp);
1199 	cache_free(ncp);
1248 	struct namecache *ncp;
1276 		ncp = dvp->v_cache_dd;
1277 		if (ncp == NULL) {
1283 		if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) {
1284 			if (ncp->nc_flag & NCF_NEGATIVE)
1287 				*vpp = ncp->nc_vp;
1289 			*vpp = ncp->nc_dvp;
1297 		cache_out_ts(ncp, tsp, ticksp);
1298 		if ((ncp->nc_flag & (NCF_ISDOTDOT | NCF_DTS)) ==
1300 			ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
1310 	LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
1312 		if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
1313 		    !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
1318 	if (ncp == NULL) {
1327 	if (!(ncp->nc_flag & NCF_NEGATIVE)) {
1329 		*vpp = ncp->nc_vp;
1330 		CTR4(KTR_VFS, "cache_lookup(%p, %s) found %p via ncp %p",
1331 		    dvp, cnp->cn_nameptr, *vpp, ncp);
1332 		SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name,
1334 		cache_out_ts(ncp, tsp, ticksp);
1346 	cache_negative_hit(ncp);
1347 	if (ncp->nc_flag & NCF_WHITE)
1350 	    ncp->nc_name);
1351 	cache_out_ts(ncp, tsp, ticksp);
1390 		error = cache_zap_rlocked_bucket(ncp, blp);
1392 		error = cache_zap_locked_vnode(ncp, dvp);
1398 	cache_free(ncp);
1534 	struct namecache *ncp;
1543 		ncp = vp->v_cache_dd;
1544 		if (ncp == NULL)
1546 		if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
1548 		MPASS(ncp->nc_dvp == vp);
1549 		blps[1] = NCP2BUCKETLOCK(ncp);
1550 		if (ncp->nc_flag & NCF_NEGATIVE)
1552 		if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp))
1558 		if (ncp == vp->v_cache_dd &&
1559 		    (ncp->nc_flag & NCF_ISDOTDOT) != 0 &&
1560 		    blps[1] == NCP2BUCKETLOCK(ncp) &&
1561 		    VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2])
1575 	struct namecache *ncp;
1582 		ncp = dvp->v_cache_dd;
1583 		if (ncp == NULL)
1585 		if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
1587 		MPASS(ncp->nc_dvp == dvp);
1588 		blps[1] = NCP2BUCKETLOCK(ncp);
1589 		if (ncp->nc_flag & NCF_NEGATIVE)
1591 		if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp))
1593 		if (ncp == dvp->v_cache_dd &&
1594 		    (ncp->nc_flag & NCF_ISDOTDOT) != 0 &&
1595 		    blps[1] == NCP2BUCKETLOCK(ncp) &&
1596 		    VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2])
1622 	struct namecache *ncp, *n2, *ndd;
1662 			if ((ncp = dvp->v_cache_dd) != NULL &&
1663 			    ncp->nc_flag & NCF_ISDOTDOT) {
1664 				KASSERT(ncp->nc_dvp == dvp,
1667 				if (ncp->nc_flag & NCF_NEGATIVE || vp == NULL) {
1668 					neglist = NCP2NEGLIST(ncp);
1673 				if (!(ncp->nc_flag & NCF_NEGATIVE)) {
1674 					TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst,
1675 					    ncp, nc_dst);
1677 					cache_negative_remove(ncp, true);
1681 					    ncp, nc_dst);
1682 					ncp->nc_flag &= ~(NCF_NEGATIVE|NCF_HOTNEGATIVE);
1684 					ncp->nc_flag &= ~(NCF_HOTNEGATIVE);
1685 					ncp->nc_flag |= NCF_NEGATIVE;
1686 					cache_negative_insert(ncp, true);
1692 				ncp->nc_vp = vp;
1708 	ncp = cache_alloc(cnp->cn_namelen, tsp != NULL);
1709 	ncp->nc_flag = flag;
1710 	ncp->nc_vp = vp;
1712 		ncp->nc_flag |= NCF_NEGATIVE;
1713 	ncp->nc_dvp = dvp;
1715 		ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
1724 	len = ncp->nc_nlen = cnp->cn_namelen;
1726 	strlcpy(ncp->nc_name, cnp->cn_nameptr, len + 1);
1747 					if (ncp->nc_flag & NCF_NEGATIVE)
1750 					if (ncp->nc_flag & NCF_NEGATIVE)
1767 		dvp->v_cache_dd = ncp;
1784 				vp->v_cache_dd = ncp;
1796 		LIST_INSERT_HEAD(&dvp->v_cache_src, ncp, nc_src);
1803 	LIST_INSERT_HEAD(ncpp, ncp, nc_hash);
1811 		TAILQ_INSERT_HEAD(&vp->v_cache_dst, ncp, nc_dst);
1812 		SDT_PROBE3(vfs, namecache, enter, done, dvp, ncp->nc_name,
1816 			ncp->nc_flag |= NCF_WHITE;
1817 		cache_negative_insert(ncp, false);
1819 		    ncp->nc_name);
1829 	cache_free(ncp);
1919 	struct namecache *ncp;
1945 		while ((ncp = LIST_FIRST(&old_nchashtbl[i])) != NULL) {
1946 			hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen,
1947 			    ncp->nc_dvp);
1948 			LIST_REMOVE(ncp, nc_hash);
1949 			LIST_INSERT_HEAD(NCHHASH(hash), ncp, nc_hash);
1964 	struct namecache *ncp, *nnp;
1978 		ncp = LIST_FIRST(&vp->v_cache_src);
1979 		if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
1981 		TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
1984 		ncp = TAILQ_FIRST(&vp->v_cache_dst);
1985 		if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
1987 		TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
1989 	ncp = vp->v_cache_dd;
1990 	if (ncp != NULL) {
1991 		KASSERT(ncp->nc_flag & NCF_ISDOTDOT,
1993 		if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
1995 		TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
2001 	TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) {
2002 		cache_free(ncp);
2013 	struct namecache *ncp, *nnp;
2023 	LIST_FOREACH_SAFE(ncp, &vp->v_cache_src, nc_src, nnp) {
2024 		if (!(ncp->nc_flag & NCF_NEGATIVE))
2026 		cache_zap_negative_locked_vnode_kl(ncp, vp);
2027 		TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
2030 	TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) {
2031 		cache_free(ncp);
2045 	struct namecache *ncp, *nnp;
2062 			LIST_FOREACH_SAFE(ncp, bucket, nc_hash, nnp) {
2063 				cache_assert_bucket_locked(ncp, RA_WLOCKED);
2064 				if (ncp->nc_dvp->v_mount != mp)
2066 				error = cache_zap_wlocked_bucket_kl(ncp, blp,
2070 				TAILQ_INSERT_HEAD(&ncps, ncp, nc_dst);
2082 	TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) {
2083 		cache_free(ncp);
2252 	struct namecache *ncp;
2258 	TAILQ_FOREACH(ncp, &((*vp)->v_cache_dst), nc_dst) {
2259 		if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
2262 	if (ncp != NULL) {
2263 		if (*buflen < ncp->nc_nlen) {
2272 		*buflen -= ncp->nc_nlen;
2273 		memcpy(buf + *buflen, ncp->nc_name, ncp->nc_nlen);
2274 		SDT_PROBE3(vfs, namecache, fullpath, hit, ncp->nc_dvp,
2275 		    ncp->nc_name, vp);
2277 		*vp = ncp->nc_dvp;
2419 	struct namecache *ncp;
2426 	TAILQ_FOREACH(ncp, &(vp->v_cache_dst), nc_dst) {
2427 		if ((ncp->nc_flag & NCF_ISDOTDOT) != 0)
2429 		ddvp = ncp->nc_dvp;
2443 	struct namecache *ncp;
2449 	TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst)
2450 		if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
2452 	if (ncp == NULL) {
2456 	l = min(ncp->nc_nlen, buflen - 1);
2457 	memcpy(buf, ncp->nc_name, l);