common/os/strsubr.c

239  * For drivers/modules that use PERMOD or outer syncqs we keep a list of
1194 	 * If there is an outer perimeter get exclusive access during
1313 	/* freeq removes us from the outer perimeter if any */
1682  * sq could be either outer or inner.
2113 		syncq_t *sq, *outer;
2122 		 * could be messages on the inner and outer/inner
2140 		if ((outer = sq->sq_outer) != NULL) {
2143 			 * We have to just wait for the outer sq_count
2145 			 * messages to enter the outer perimeter, this
2150 			 * thread holding the outer perimeter exclusively.
2152 			 * exclusive holder of the outer perimeter to drain
2153 			 * the inner and outer syncqs. But we will not depend
2157 			wait_syncq(outer);
2438 	/* Default outer perimeter concurrency */
2444 			/* No outer perimeter */
2490 	syncq_t	*sq, *outer;
2510 	outer = sq->sq_outer;
2511 	if (outer != NULL) {
2512 		ASSERT(wq->q_syncq->sq_outer == outer);
2513 		outer_remove(outer, rq->q_syncq);
2515 			outer_remove(outer, wq->q_syncq);
2613 		 * does not have an outer perimeter associated with it.
2620 		outer = dmp->dm_sq;
2622 		ASSERT(outer->sq_outer == NULL);
2623 		outer_insert(outer, rq->q_syncq);
2625 			outer_insert(outer, wq->q_syncq);
3409  * outer perimeter, and free the syncq(s) if they are not the "attached" syncq.
3415 	syncq_t *sq, *outer;
3447 	outer = qp->q_syncq->sq_outer;
3448 	if (outer != NULL) {
3449 		outer_remove(outer, qp->q_syncq);
3451 			outer_remove(outer, wqp->q_syncq);
4480  *  - Messages on an outer perimeter (syncq) that have a reference to the
4490  * the syncq or the outer perimeter.
5006  * This routine is used for both inner and outer syncqs.
5063  * If "isouter" is set then the syncq is assumed to be an outer perimeter
5410 	 * We have to do this before entering the outer perimeter in order
5435 	/* Check if we need to enter the outer perimeter */
5438 		 * We have to enter the outer perimeter exclusively before
5539 			/* Check if we need to exit the outer perimeter */
5552 	/* Check if we need to exit the outer perimeter */
5691  * The outer syncq uses the fields and flags in the syncq slightly
5694  *			writers at the outer perimeter to prevent the set of
5695  *			inner syncqs that belong to the outer perimeter from
5701  *			outer perimeter.
5709  * Get write access at the outer perimeter.
5713  * Waits until "flags" is no longer set in the outer to prevent multiple
5717  * Increases sq_count on the outer syncq to keep away outer_insert/remove
5725 outer_enter(syncq_t *outer, uint16_t flags)
5731 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5732 	    outer->sq_oprev != NULL);
5736 	mutex_enter(SQLOCK(outer));
5737 	while (outer->sq_flags & flags) {
5738 		outer->sq_flags |= SQ_WANTWAKEUP;
5739 		cv_wait(&outer->sq_wait, SQLOCK(outer));
5742 	ASSERT(!(outer->sq_flags & SQ_WRITER));
5743 	outer->sq_flags |= SQ_WRITER;
5744 	outer->sq_count++;
5745 	ASSERT(outer->sq_count != 0);	/* wraparound */
5749 	 * the SQLOCK on the outer syncq. This ensures that the changing
5750 	 * of SQ_WRITER is atomic under the outer SQLOCK.
5752 	for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5763 	mutex_exit(SQLOCK(outer));
5777 		for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5797 		 * outer perimeter.
5799 		mutex_enter(SQLOCK(outer));
5800 		if (outer->sq_flags & (flags & ~SQ_WRITER)) {
5801 			mutex_exit(SQLOCK(outer));
5802 			outer_exit(outer);
5805 		mutex_exit(SQLOCK(outer));
5810  * Drop the write access at the outer perimeter.
5815 outer_exit(syncq_t *outer)
5821 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5822 	    outer->sq_oprev != NULL);
5823 	ASSERT(MUTEX_NOT_HELD(SQLOCK(outer)));
5827 	 * drop the write access at the outer perimeter by holding
5828 	 * SQLOCK(outer) across all the dropsq calls and the resetting of
5830 	 * This defines a locking order between the outer perimeter
5833 	mutex_enter(SQLOCK(outer));
5834 	flags = outer->sq_flags;
5835 	ASSERT(outer->sq_flags & SQ_WRITER);
5837 		write_now(outer);
5838 		flags = outer->sq_flags;
5844 	 * After dropping SQ_WRITER on the outer syncq we empty all the
5848 	for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5850 	ASSERT(!(outer->sq_flags & SQ_QUEUED));
5853 		outer->sq_flags = flags;
5854 		mutex_exit(SQLOCK(outer));
5855 		for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5857 		mutex_enter(SQLOCK(outer));
5858 		flags = outer->sq_flags;
5862 		cv_broadcast(&outer->sq_wait);
5864 	outer->sq_flags = flags;
5865 	ASSERT(outer->sq_count > 0);
5866 	outer->sq_count--;
5867 	mutex_exit(SQLOCK(outer));
5871  * Add another syncq to an outer perimeter.
5872  * Block out all other access to the outer perimeter while it is being
5879 outer_insert(syncq_t *outer, syncq_t *sq)
5881 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5882 	    outer->sq_oprev != NULL);
5884 	    sq->sq_oprev == NULL);	/* Can't be in an outer perimeter */
5886 	/* Get exclusive access to the outer perimeter list */
5887 	blocksq(outer, SQ_BLOCKED, 0);
5888 	ASSERT(outer->sq_flags & SQ_BLOCKED);
5889 	ASSERT(!(outer->sq_flags & SQ_WRITER));
5892 	sq->sq_outer = outer;
5893 	outer->sq_onext->sq_oprev = sq;
5894 	sq->sq_onext = outer->sq_onext;
5895 	outer->sq_onext = sq;
5896 	sq->sq_oprev = outer;
5898 	unblocksq(outer, SQ_BLOCKED, 1);
5902  * Remove a syncq from an outer perimeter.
5903  * Block out all other access to the outer perimeter while it is being
5910 outer_remove(syncq_t *outer, syncq_t *sq)
5912 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5913 	    outer->sq_oprev != NULL);
5914 	ASSERT(sq->sq_outer == outer);
5916 	/* Get exclusive access to the outer perimeter list */
5917 	blocksq(outer, SQ_BLOCKED, 0);
5918 	ASSERT(outer->sq_flags & SQ_BLOCKED);
5919 	ASSERT(!(outer->sq_flags & SQ_WRITER));
5927 	unblocksq(outer, SQ_BLOCKED, 1);
5931  * Queue a deferred qwriter(OUTER) callback for this outer perimeter.
5932  * If this is the first callback for this outer perimeter then add
5933  * this outer perimeter to the list of outer perimeters that
5936  * Increments sq_count in the outer syncq to prevent the membership
5937  * of the outer perimeter (in terms of inner syncqs) to change while
5941 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp)
5943 	ASSERT(MUTEX_HELD(SQLOCK(outer)));
5948 	outer->sq_count++;	/* Decremented when dequeued */
5949 	ASSERT(outer->sq_count != 0);	/* Wraparound */
5950 	if (outer->sq_evhead == NULL) {
5952 		outer->sq_evhead = outer->sq_evtail = mp;
5953 		outer->sq_flags |= SQ_EVENTS;
5954 		mutex_exit(SQLOCK(outer));
5957 		    (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP);
5959 		ASSERT(outer->sq_flags & SQ_EVENTS);
5960 		outer->sq_evtail->b_next = mp;
5961 		outer->sq_evtail = mp;
5962 		mutex_exit(SQLOCK(outer));
5967  * Try and upgrade to write access at the outer perimeter. If this can
5979 	syncq_t	*osq, *sq, *outer;
5984 	outer = osq->sq_outer;
5985 	if (outer == NULL)
5986 		panic("qwriter(PERIM_OUTER): no outer perimeter");
5987 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5988 	    outer->sq_oprev != NULL);
5990 	mutex_enter(SQLOCK(outer));
5991 	flags = outer->sq_flags;
6004 		 * The queueing is atomic under SQLOCK(outer) in order
6006 		 * queue_writer will drop the outer SQLOCK
6016 				 * The outer could have been SQ_BLOCKED thus
6025 		queue_writer(outer, func, q, mp);
6029 	 * We are half-way to exclusive access to the outer perimeter.
6033 	outer->sq_count++;
6034 	ASSERT(outer->sq_count != 0);	/* wraparound */
6042 	 * the SQLOCK on the outer syncq. This ensures that the changing
6043 	 * of SQ_WRITER is atomic under the outer SQLOCK.
6046 	for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
6062 		 * Some other thread has a read claim on the outer perimeter.
6071 		 * on this outer perimeter.
6073 		outer->sq_flags = flags;
6074 		queue_writer(outer, func, q, mp);
6076 		mutex_enter(SQLOCK(outer));
6077 		ASSERT(outer->sq_count > 0);
6078 		outer->sq_count--;
6079 		ASSERT(outer->sq_flags & SQ_WRITER);
6080 		flags = outer->sq_flags;
6084 			cv_broadcast(&outer->sq_wait);
6086 		outer->sq_flags = flags;
6087 		mutex_exit(SQLOCK(outer));
6090 		outer->sq_flags = flags;
6091 		mutex_exit(SQLOCK(outer));
6097 	outer_exit(outer);
6101  * Dequeue all writer callbacks from the outer perimeter and run them.
6104 write_now(syncq_t *outer)
6110 	ASSERT(MUTEX_HELD(SQLOCK(outer)));
6111 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
6112 	    outer->sq_oprev != NULL);
6113 	while ((mp = outer->sq_evhead) != NULL) {
6115 		 * queues cannot be placed on the queuelist on the outer
6118 		ASSERT(!(outer->sq_flags & SQ_MESSAGES));
6119 		ASSERT((outer->sq_flags & SQ_EVENTS));
6121 		outer->sq_evhead = mp->b_next;
6122 		if (outer->sq_evhead == NULL) {
6123 			outer->sq_evtail = NULL;
6124 			outer->sq_flags &= ~SQ_EVENTS;
6126 		ASSERT(outer->sq_count != 0);
6127 		outer->sq_count--;	/* Incremented when enqueued. */
6128 		mutex_exit(SQLOCK(outer));
6145 		mutex_enter(SQLOCK(outer));
6147 	ASSERT(MUTEX_HELD(SQLOCK(outer)));
6666 	 * All outer pointers are set, or none of them are
7391 	/* Can not handle exclusive entry at outer perimeter */
8299 qwriter_outer_service(syncq_t *outer)
8302 	 * Note that SQ_WRITER is used on the outer perimeter
8306 	outer_enter(outer, SQ_BLOCKED|SQ_WRITER);
8310 	 * to block entering the outer perimeter.
8315 	outer_exit(outer);