Lines Matching refs:cp
263 mcache_t *cp = NULL;
295 cp = (mcache_t *)
297 pbuf = (void **)((intptr_t)cp - sizeof (void *));
312 cp->mc_align = align;
313 cp->mc_slab_alloc = allocfn;
314 cp->mc_slab_free = freefn;
315 cp->mc_slab_audit = auditfn;
316 cp->mc_slab_log = logfn;
317 cp->mc_slab_notify = notifyfn;
318 cp->mc_private = need_zone ? cp : arg;
319 cp->mc_bufsize = bufsize;
320 cp->mc_flags = (flags & MCF_FLAGS_MASK) | mcache_flags;
322 (void) snprintf(cp->mc_name, sizeof (cp->mc_name), "mcache.%s", name);
324 (void) snprintf(lck_name, sizeof (lck_name), "%s.cpu", cp->mc_name);
325 cp->mc_cpu_lock_grp_attr = lck_grp_attr_alloc_init();
326 cp->mc_cpu_lock_grp = lck_grp_alloc_init(lck_name,
327 cp->mc_cpu_lock_grp_attr);
328 cp->mc_cpu_lock_attr = lck_attr_alloc_init();
343 if ((cp->mc_slab_zone = zinit(chunksize, 64 * 1024 * ncpu,
344 PAGE_SIZE, cp->mc_name)) == NULL)
346 zone_change(cp->mc_slab_zone, Z_EXPAND, TRUE);
348 cp->mc_chunksize = chunksize;
353 (void) snprintf(lck_name, sizeof (lck_name), "%s.bkt", cp->mc_name);
354 cp->mc_bkt_lock_grp_attr = lck_grp_attr_alloc_init();
355 cp->mc_bkt_lock_grp = lck_grp_alloc_init(lck_name,
356 cp->mc_bkt_lock_grp_attr);
357 cp->mc_bkt_lock_attr = lck_attr_alloc_init();
358 lck_mtx_init(&cp->mc_bkt_lock, cp->mc_bkt_lock_grp,
359 cp->mc_bkt_lock_attr);
361 (void) snprintf(lck_name, sizeof (lck_name), "%s.sync", cp->mc_name);
362 cp->mc_sync_lock_grp_attr = lck_grp_attr_alloc_init();
363 cp->mc_sync_lock_grp = lck_grp_alloc_init(lck_name,
364 cp->mc_sync_lock_grp_attr);
365 cp->mc_sync_lock_attr = lck_attr_alloc_init();
366 lck_mtx_init(&cp->mc_sync_lock, cp->mc_sync_lock_grp,
367 cp->mc_sync_lock_attr);
372 cp->cache_bkttype = btp;
379 mcache_cpu_t *ccp = &cp->mc_cpu[c];
382 lck_mtx_init(&ccp->cc_lock, cp->mc_cpu_lock_grp,
383 cp->mc_cpu_lock_attr);
389 mcache_cache_bkt_enable(cp);
394 LIST_INSERT_HEAD(&mcache_head, cp, mc_list);
405 if (cp->mc_flags & MCF_DEBUG) {
408 arg, bufsize, cp->mc_align, chunksize, btp->bt_bktsize);
410 return (cp);
422 mcache_alloc_ext(mcache_t *cp, mcache_obj_t **list, unsigned int num, int wait)
441 ccp = MCACHE_CPU(cp);
472 if (!(cp->mc_flags & MCF_NOLEAKLOG) &&
473 cp->mc_slab_log != NULL)
474 (*cp->mc_slab_log)(num, *top, TRUE);
476 if (cp->mc_flags & MCF_DEBUG)
505 bkt = mcache_bkt_alloc(cp, &cp->mc_full, NULL);
508 mcache_bkt_free(cp, &cp->mc_empty,
522 need -= (*cp->mc_slab_alloc)(cp->mc_private, &list, need, wait);
530 atomic_add_32(&cp->mc_wretry_cnt, 1);
533 !mcache_bkt_isempty(cp)) {
536 atomic_add_32(&cp->mc_nwretry_cnt, 1);
539 atomic_add_32(&cp->mc_nwfail_cnt, 1);
543 if (!(cp->mc_flags & MCF_NOLEAKLOG) && cp->mc_slab_log != NULL)
544 (*cp->mc_slab_log)((num - need), *top, TRUE);
546 if (!(cp->mc_flags & MCF_DEBUG))
550 if (cp->mc_flags & MCF_DEBUG) {
566 panic("mcache_alloc_ext: %s cp %p corrupted list "
567 "(got %d actual %d)\n", cp->mc_name,
568 (void *)cp, num - need, n);
573 if ((cp->mc_flags & MCF_DEBUG) && cp->mc_slab_audit != NULL)
574 (*cp->mc_slab_audit)(cp->mc_private, *top, TRUE);
583 mcache_alloc(mcache_t *cp, int wait)
587 (void) mcache_alloc_ext(cp, &buf, 1, wait);
592 mcache_waiter_inc(mcache_t *cp)
594 atomic_add_32(&cp->mc_waiter_cnt, 1);
598 mcache_waiter_dec(mcache_t *cp)
600 atomic_add_32(&cp->mc_waiter_cnt, -1);
604 mcache_bkt_isempty(mcache_t *cp)
611 return (cp->mc_full.bl_total == 0);
618 mcache_notify(mcache_t *cp, u_int32_t event)
620 if (cp->mc_slab_notify != NULL)
621 (*cp->mc_slab_notify)(cp->mc_private, event);
630 mcache_t *cp = arg;
632 mcache_bkt_purge(cp);
640 lck_mtx_lock_spin(&cp->mc_sync_lock);
641 cp->mc_enable_cnt++;
642 lck_mtx_unlock(&cp->mc_sync_lock);
647 mcache_purge_cache(mcache_t *cp)
653 if (cp->mc_flags & MCF_NOCPUCACHE)
656 lck_mtx_lock_spin(&cp->mc_sync_lock);
657 if (cp->mc_purge_cnt > 0) {
658 lck_mtx_unlock(&cp->mc_sync_lock);
661 cp->mc_purge_cnt++;
662 lck_mtx_unlock(&cp->mc_sync_lock);
664 mcache_dispatch(mcache_purge, cp);
673 mcache_free(mcache_t *cp, void *buf)
676 mcache_free_ext(cp, (mcache_obj_t *)buf);
683 mcache_free_ext(mcache_t *cp, mcache_obj_t *list)
685 mcache_cpu_t *ccp = MCACHE_CPU(cp);
690 if (!(cp->mc_flags & MCF_NOLEAKLOG) && cp->mc_slab_log != NULL)
691 (*cp->mc_slab_log)(0, list, FALSE);
694 if ((cp->mc_flags & MCF_DEBUG) && cp->mc_slab_audit != NULL)
695 (*cp->mc_slab_audit)(cp->mc_private, list, FALSE);
728 if (cp->mc_waiter_cnt > 0)
729 mcache_notify(cp, MCN_RETRYALLOC);
755 bkt = mcache_bkt_alloc(cp, &cp->mc_empty, &btp);
758 mcache_bkt_free(cp, &cp->mc_full,
792 mcache_bkt_free(cp, &cp->mc_empty, bkt);
805 if (cp->mc_waiter_cnt > 0)
806 mcache_notify(cp, MCN_RETRYALLOC);
809 (*cp->mc_slab_free)(cp->mc_private, list,
810 (cp->mc_flags & MCF_DEBUG) || cp->mc_purge_cnt);
817 mcache_destroy(mcache_t *cp)
822 LIST_REMOVE(cp, mc_list);
825 mcache_bkt_purge(cp);
831 cp->mc_slab_alloc = NULL;
832 cp->mc_slab_free = NULL;
833 cp->mc_slab_audit = NULL;
835 lck_attr_free(cp->mc_bkt_lock_attr);
836 lck_grp_free(cp->mc_bkt_lock_grp);
837 lck_grp_attr_free(cp->mc_bkt_lock_grp_attr);
839 lck_attr_free(cp->mc_cpu_lock_attr);
840 lck_grp_free(cp->mc_cpu_lock_grp);
841 lck_grp_attr_free(cp->mc_cpu_lock_grp_attr);
843 lck_attr_free(cp->mc_sync_lock_attr);
844 lck_grp_free(cp->mc_sync_lock_grp);
845 lck_grp_attr_free(cp->mc_sync_lock_grp_attr);
853 * if (cp->mc_slab_zone != NULL) {
854 * zdestroy(cp->mc_slab_zone);
855 * cp->mc_slab_zone = NULL;
860 pbuf = (void **)((intptr_t)cp - sizeof (void *));
872 mcache_t *cp = arg;
875 size_t rsize = P2ROUNDUP(cp->mc_bufsize, sizeof (u_int64_t));
876 u_int32_t flags = cp->mc_flags;
887 if (cp->mc_align != 1 && cp->mc_align != sizeof (u_int64_t))
888 offset = cp->mc_align;
892 buf = zalloc(cp->mc_slab_zone);
894 buf = zalloc_noblock(cp->mc_slab_zone);
919 ((intptr_t)buf + cp->mc_chunksize));
928 VERIFY(((intptr_t)base + offset + cp->mc_bufsize) <=
929 ((intptr_t)buf + cp->mc_chunksize));
949 mcache_t *cp = arg;
952 size_t rsize = P2ROUNDUP(cp->mc_bufsize, sizeof (u_int64_t));
953 u_int32_t flags = cp->mc_flags;
962 if (cp->mc_align != 1 && cp->mc_align != sizeof (u_int64_t))
963 offset = cp->mc_align;
978 ((intptr_t)*pbuf + cp->mc_chunksize));
983 VERIFY(((intptr_t)base + offset + cp->mc_bufsize) <=
984 ((intptr_t)*pbuf + cp->mc_chunksize));
985 zfree(cp->mc_slab_zone, *pbuf);
999 mcache_t *cp = arg;
1001 size_t rsize = P2ROUNDUP(cp->mc_bufsize, sizeof (u_int64_t));
1009 if (cp->mc_align != 1 && cp->mc_align != sizeof (u_int64_t))
1010 offset = cp->mc_align;
1023 ((intptr_t)*pbuf + cp->mc_chunksize));
1054 mcache_bkt_alloc(mcache_t *cp, mcache_bktlist_t *blp, mcache_bkttype_t **btp)
1058 if (!MCACHE_LOCK_TRY(&cp->mc_bkt_lock)) {
1064 MCACHE_LOCK(&cp->mc_bkt_lock);
1065 cp->mc_bkt_contention++;
1076 *btp = cp->cache_bkttype;
1078 MCACHE_UNLOCK(&cp->mc_bkt_lock);
1087 mcache_bkt_free(mcache_t *cp, mcache_bktlist_t *blp, mcache_bkt_t *bkt)
1089 MCACHE_LOCK(&cp->mc_bkt_lock);
1095 MCACHE_UNLOCK(&cp->mc_bkt_lock);
1102 mcache_cache_bkt_enable(mcache_t *cp)
1107 if (cp->mc_flags & MCF_NOCPUCACHE)
1111 ccp = &cp->mc_cpu[cpu];
1113 ccp->cc_bktsize = cp->cache_bkttype->bt_bktsize;
1122 mcache_bkt_purge(mcache_t *cp)
1130 ccp = &cp->mc_cpu[cpu];
1134 btp = cp->cache_bkttype;
1148 mcache_bkt_destroy(cp, btp, bp, objs);
1150 mcache_bkt_destroy(cp, btp, pbp, pobjs);
1157 mcache_bkt_ws_update(cp);
1158 mcache_bkt_ws_update(cp);
1160 mcache_bkt_ws_reap(cp);
1168 mcache_bkt_destroy(mcache_t *cp, mcache_bkttype_t *btp, mcache_bkt_t *bkt,
1174 if (cp->mc_flags & MCF_DEBUG) {
1188 panic("mcache_bkt_destroy: %s cp %p corrupted "
1190 cp->mc_name, (void *)cp, (void *)bkt,
1196 (*cp->mc_slab_free)(cp->mc_private, top,
1197 (cp->mc_flags & MCF_DEBUG) || cp->mc_purge_cnt);
1206 mcache_bkt_ws_update(mcache_t *cp)
1208 MCACHE_LOCK(&cp->mc_bkt_lock);
1210 cp->mc_full.bl_reaplimit = cp->mc_full.bl_min;
1211 cp->mc_full.bl_min = cp->mc_full.bl_total;
1212 cp->mc_empty.bl_reaplimit = cp->mc_empty.bl_min;
1213 cp->mc_empty.bl_min = cp->mc_empty.bl_total;
1215 MCACHE_UNLOCK(&cp->mc_bkt_lock);
1222 mcache_bkt_ws_reap(mcache_t *cp)
1228 reap = MIN(cp->mc_full.bl_reaplimit, cp->mc_full.bl_min);
1230 (bkt = mcache_bkt_alloc(cp, &cp->mc_full, &btp)) != NULL)
1231 mcache_bkt_destroy(cp, btp, bkt, btp->bt_bktsize);
1233 reap = MIN(cp->mc_empty.bl_reaplimit, cp->mc_empty.bl_min);
1235 (bkt = mcache_bkt_alloc(cp, &cp->mc_empty, &btp)) != NULL)
1236 mcache_bkt_destroy(cp, btp, bkt, 0);
1279 mcache_cache_reap(mcache_t *cp)
1281 mcache_bkt_ws_reap(cp);
1288 mcache_cache_update(mcache_t *cp)
1295 mcache_bkt_ws_update(cp);
1303 lck_mtx_lock_spin(&cp->mc_sync_lock);
1304 if (!(cp->mc_flags & MCF_NOCPUCACHE) && cp->mc_enable_cnt)
1306 lck_mtx_unlock(&cp->mc_sync_lock);
1308 MCACHE_LOCK(&cp->mc_bkt_lock);
1315 if ((unsigned int)cp->mc_chunksize < cp->cache_bkttype->bt_maxbuf &&
1316 (int)(cp->mc_bkt_contention - cp->mc_bkt_contention_prev) >
1320 cp ->mc_bkt_contention_prev = cp->mc_bkt_contention;
1321 MCACHE_UNLOCK(&cp->mc_bkt_lock);
1324 mcache_dispatch(mcache_cache_bkt_resize, cp);
1326 mcache_dispatch(mcache_cache_enable, cp);
1338 mcache_t *cp = arg;
1339 mcache_bkttype_t *btp = cp->cache_bkttype;
1341 if ((unsigned int)cp->mc_chunksize < btp->bt_maxbuf) {
1342 mcache_bkt_purge(cp);
1349 MCACHE_LOCK(&cp->mc_bkt_lock);
1350 cp->cache_bkttype = ++btp;
1351 cp ->mc_bkt_contention_prev = cp->mc_bkt_contention + INT_MAX;
1352 MCACHE_UNLOCK(&cp->mc_bkt_lock);
1354 mcache_cache_enable(cp);
1364 mcache_t *cp = arg;
1366 lck_mtx_lock_spin(&cp->mc_sync_lock);
1367 cp->mc_purge_cnt = 0;
1368 cp->mc_enable_cnt = 0;
1369 lck_mtx_unlock(&cp->mc_sync_lock);
1371 mcache_cache_bkt_enable(cp);
1390 mcache_t *cp;
1393 LIST_FOREACH(cp, &mcache_head, mc_list) {
1394 func(cp);
1407 mcache_buffer_log(mcache_audit_t *mca, void *addr, mcache_t *cp)
1410 mca->mca_cache = cp;